Skip to main content

reflex/
indexer.rs

1//! Indexing engine for parsing source code
2//!
3//! The indexer scans the project directory, parses source files using Tree-sitter,
4//! and builds the symbol/token cache for fast querying.
5
6use anyhow::{Context, Result};
7use ignore::WalkBuilder;
8use indicatif::{ProgressBar, ProgressDrawTarget, ProgressStyle};
9use rayon::prelude::*;
10use std::collections::HashMap;
11use std::path::{Path, PathBuf};
12use std::sync::atomic::{AtomicU64, Ordering};
13use std::sync::{Arc, Mutex};
14use std::time::Instant;
15
16use crate::cache::{CacheManager, WALK_SEQ_GAP};
17use crate::content_store::ContentWriter;
18use crate::models::{IndexConfig, IndexMode, IndexStats, Language};
19#[cfg(unix)]
20use crate::output;
21use crate::parsers::c::CDependencyExtractor;
22use crate::parsers::cpp::CppDependencyExtractor;
23use crate::parsers::csharp::CSharpDependencyExtractor;
24use crate::parsers::go::GoDependencyExtractor;
25use crate::parsers::java::JavaDependencyExtractor;
26use crate::parsers::kotlin::KotlinDependencyExtractor;
27use crate::parsers::php::PhpDependencyExtractor;
28use crate::parsers::python::PythonDependencyExtractor;
29use crate::parsers::ruby::RubyDependencyExtractor;
30use crate::parsers::rust::RustDependencyExtractor;
31use crate::parsers::svelte::SvelteDependencyExtractor;
32use crate::parsers::swift_imports::SwiftDependencyExtractor;
33use crate::parsers::typescript::TypeScriptDependencyExtractor;
34use crate::parsers::vue::VueDependencyExtractor;
35use crate::parsers::zig::ZigDependencyExtractor;
36use crate::parsers::{DependencyExtractor, ExportInfo, ImportInfo};
37use crate::trigram_build::{TrigramIndexBuilder, TrigramRun};
38
39/// Progress callback type: (current_file_count, total_file_count, status_message)
40/// Uses Arc to allow cloning for multi-threaded progress updates
41pub type ProgressCallback = Arc<dyn Fn(usize, usize, String) + Send + Sync>;
42
43/// Result of processing a single file (used for parallel processing)
44struct FileProcessingResult {
45    hash: String,
46    content: String,
47    language: Language,
48    line_count: usize,
49    /// On-disk size and mtime, taken BEFORE the read so a write that lands
50    /// between the two is caught by the next status check, not hidden by it.
51    size: u64,
52    mtime_ns: i64,
53    /// Imports, re-exports and package memberships, when they were extracted
54    /// this run.
55    imports: Option<Extracted>,
56    /// The file's trigram postings, extracted in the pool (no file id yet).
57    trigram_run: TrigramRun,
58}
59
60/// `statistics` key of the resolver config digest the dependency rows were
61/// resolved with (see [`crate::dependency_resolve::ResolverConfigs::digest`]).
62const RESOLVER_DIGEST_KEY: &str = "resolver_config_digest";
63
64/// `statistics` key of the manifest generation meta.db's rows go with. A manifest
65/// of another generation means a run stopped between the two commits.
66pub const INDEX_GENERATION_KEY: &str = "index_generation";
67
68/// File in `.reflex/` whose mtime is an index run's start (see `run_marker`).
69const RUN_MARKER: &str = ".index-run";
70
71/// What a run found for one discovered file, compared with its `files` row.
72#[derive(Debug, Clone, Copy, PartialEq, Eq)]
73enum FileStatus {
74    /// Same size and mtime as stored: not read.
75    Unchanged,
76    /// Different stat, same bytes (a `touch`, or an edit reverted).
77    Touched,
78    /// Different bytes.
79    Modified,
80    /// No row.
81    Added,
82}
83
84/// One file the build read, in walk order.
85struct IndexedFile {
86    /// Position in the discovery lists.
87    index: usize,
88    hash: String,
89    language: Language,
90    line_count: usize,
91    size: u64,
92    mtime_ns: i64,
93    /// Imports, re-exports and package memberships, when they were extracted
94    /// this run.
95    imports: Option<Extracted>,
96}
97
98/// One added or modified file a delta update reads.
99struct Rewrite {
100    rel: String,
101    /// Size from the walk's stat (the merge-limit check before any read).
102    size: u64,
103    walk_seq: i64,
104}
105
106/// The dirty flags a change set records: the rewritten paths that are dirty, the
107/// touched rows `(id, size, mtime_ns, dirty)`, the rows whose flag flips, and the
108/// branch's flag.
109struct DirtyFlags {
110    rewrites: std::collections::HashSet<String>,
111    touched: Vec<(i64, u64, i64, bool)>,
112    flips: Vec<(i64, bool)>,
113    git_dirty: bool,
114}
115
116/// A delta update's change set (see `Indexer::publish_delta`): `rfx index` builds
117/// it from a full walk, `update_paths` from the named paths. Rows and store
118/// entries it does not name stay as they are.
119struct DeltaChanges<'a> {
120    root: &'a Path,
121    /// Added and modified files, in walk order.
122    rewrites: Vec<Rewrite>,
123    /// Rows of files gone from the index: `(path, id)`.
124    deleted: Vec<(String, i64)>,
125    /// Rows whose walk position moved: `(id, walk_seq)`.
126    walk_moves: Vec<(i64, i64)>,
127    /// The dirty flags, asked for only when meta.db is written (a library update's
128    /// `git status` runs while the stores are written).
129    dirty: Box<dyn FnOnce() -> DirtyFlags + 'a>,
130    /// The rows of the rewritten paths that have one.
131    stored: HashMap<String, crate::meta_update::StoredFile>,
132    /// Re-sync every branch row (the branch may not be the one last synced);
133    /// otherwise only the rewritten rows are set.
134    sync_all_branch_rows: bool,
135    /// Return the statistics `rfx index` prints (per-language counts: a scan of
136    /// every row); otherwise the counts and sizes only.
137    full_stats: bool,
138    /// Files in the index after the update (unreadable rewrites not subtracted).
139    live_files: usize,
140    /// (new, modified, unchanged) of the files not rewritten, against the branch's
141    /// own hashes.
142    breakdown_rest: (usize, usize, usize),
143    /// The branch's own hashes of the rewritten paths.
144    branch_hashes: HashMap<String, String>,
145    branch: &'a str,
146    commit: Option<&'a str>,
147    resolver_configs: &'a crate::dependency_resolve::ResolverConfigs,
148    run_start: std::time::SystemTime,
149    generation: u64,
150    pool: &'a rayon::ThreadPool,
151    deleted_file_count: usize,
152    /// (too large, bytes too large, binary)
153    skipped: (usize, u64, usize),
154}
155
156/// What `Indexer::process_file` reads from the run.
157struct ProcessCtx<'a> {
158    root: &'a Path,
159    rels: &'a [String],
160    stored: &'a HashMap<String, crate::meta_update::StoredFile>,
161    run_start: std::time::SystemTime,
162    full_deps: bool,
163    tsconfigs: &'a HashMap<PathBuf, crate::parsers::tsconfig::PathAliasMap>,
164}
165
166/// Inputs of `Indexer::refresh_unchanged`.
167struct RefreshUnchanged<'a> {
168    rels: &'a [String],
169    metas: &'a [Option<crate::cache::FileStat>],
170    status: &'a [FileStatus],
171    /// Each walked path's row (every one has a row: nothing was added).
172    rows: &'a [Option<&'a crate::meta_update::StoredFile>],
173    existing_hashes: &'a BranchHashes<'a>,
174    dirty_paths: &'a std::collections::HashSet<String>,
175    branch: &'a str,
176    /// The branch's rows already name every file (the last run synced them).
177    in_sync: bool,
178    commit: Option<&'a str>,
179    /// `Some(dirty)` inside git.
180    git_dirty: Option<bool>,
181    run_start: std::time::SystemTime,
182    /// (too large, bytes too large, binary)
183    skipped: (usize, u64, usize),
184}
185
186/// Inputs of `Indexer::refresh_named`.
187struct RefreshNamed<'a> {
188    touched: &'a [(i64, u64, i64, bool)],
189    flips: &'a [(i64, bool)],
190    walk_moves: &'a [(i64, i64)],
191    branch: &'a str,
192    commit: Option<&'a str>,
193    git_dirty: bool,
194    live_files: usize,
195    /// (too large, bytes too large, binary)
196    skipped: (usize, u64, usize),
197}
198
199/// Inputs of `Indexer::write_meta`.
200struct MetaWrite<'a> {
201    root: &'a Path,
202    rels: &'a [String],
203    /// Discovery indices of the files in the new snapshot, in walk order.
204    present: &'a [usize],
205    /// Files read this run, in walk order (a subset of `present`).
206    written: &'a [IndexedFile],
207    metas: &'a [Option<crate::cache::FileStat>],
208    run_start: std::time::SystemTime,
209    stored: &'a HashMap<String, crate::meta_update::StoredFile>,
210    status: &'a [FileStatus],
211    dirty_paths: &'a std::collections::HashSet<String>,
212    branch: &'a str,
213    commit: Option<&'a str>,
214    resolver_configs: &'a crate::dependency_resolve::ResolverConfigs,
215    full_deps: bool,
216    /// Generation of the manifest these rows go with.
217    generation: u64,
218}
219
220/// (new, modified, unchanged) of `(path, hash)` pairs against a branch's hashes.
221/// The branch's own hash of each path: its `file_branches` rows, or, for the
222/// branch the last run synced, the `files` rows themselves (the same values,
223/// without loading them).
224enum BranchHashes<'a> {
225    Synced(&'a HashMap<String, crate::meta_update::StoredFile>),
226    Loaded(HashMap<String, String>),
227}
228
229impl BranchHashes<'_> {
230    fn get(&self, path: &str) -> Option<&str> {
231        match self {
232            Self::Synced(stored) => stored.get(path).map(|row| row.hash.as_str()),
233            Self::Loaded(hashes) => hashes.get(path).map(String::as_str),
234        }
235    }
236}
237
238fn breakdown<'a>(
239    files: impl Iterator<Item = (&'a str, &'a str)>,
240    existing_hashes: &BranchHashes<'_>,
241) -> (usize, usize, usize) {
242    let (mut new, mut modified, mut unchanged) = (0, 0, 0);
243    for (path, hash) in files {
244        match existing_hashes.get(path) {
245            None => new += 1,
246            Some(old) if old != hash => modified += 1,
247            _ => unchanged += 1,
248        }
249    }
250    (new, modified, unchanged)
251}
252
253/// Set `files.vendored` from `rules` for the rows in `ids`, or for every row
254/// (`None`: the rules may have changed). Returns how many rows changed.
255fn refresh_vendored(
256    tx: &rusqlite::Connection,
257    rules: &crate::vendor::VendorRules,
258    ids: Option<&[i64]>,
259) -> Result<usize> {
260    let mut changed: Vec<(i64, bool)> = Vec::new();
261    {
262        let mut visit = |id: i64, path: &str, old: bool| {
263            let new = rules.is_vendored(path, crate::models::Language::from_path(Path::new(path)));
264            if new != old {
265                changed.push((id, new));
266            }
267        };
268        match ids {
269            None => {
270                let mut stmt = tx.prepare("SELECT id, path, vendored FROM files")?;
271                let rows = stmt.query_map([], |r| {
272                    Ok((
273                        r.get::<_, i64>(0)?,
274                        r.get::<_, String>(1)?,
275                        r.get::<_, bool>(2)?,
276                    ))
277                })?;
278                for row in rows {
279                    let (id, path, old) = row?;
280                    visit(id, &path, old);
281                }
282            }
283            Some(ids) => {
284                let mut stmt = tx.prepare("SELECT path, vendored FROM files WHERE id = ?")?;
285                for &id in ids {
286                    let (path, old): (String, bool) =
287                        stmt.query_row([id], |r| Ok((r.get(0)?, r.get(1)?)))?;
288                    visit(id, &path, old);
289                }
290            }
291        }
292    }
293    let mut stmt = tx.prepare("UPDATE files SET vendored = ? WHERE id = ?")?;
294    for (id, vendored) in &changed {
295        stmt.execute(rusqlite::params![vendored, id])?;
296    }
297    Ok(changed.len())
298}
299
300/// Resolve again every stored import (other than External/Stdlib) and every
301/// export of the files not in `skip`, updating the rows whose target changed.
302/// Package-keyed rows are skipped: their key depends on the import and the
303/// module config only, and which files a package holds is `package_members`.
304/// Returns how many rows changed.
305fn reresolve(
306    tx: &rusqlite::Connection,
307    ctx: &crate::dependency_resolve::ResolverContext<'_>,
308    resolver: &crate::dependency::PathResolver,
309    skip: &std::collections::HashSet<i64>,
310) -> Result<usize> {
311    let mut changed: Vec<(i64, Option<i64>)> = Vec::new();
312    {
313        let mut stmt = tx.prepare(
314            "SELECT d.id, d.file_id, f.path, d.imported_path, d.resolved_file_id
315             FROM file_dependencies d JOIN files f ON f.id = d.file_id
316             WHERE d.import_type NOT IN ('external', 'stdlib')
317               AND d.resolved_package IS NULL",
318        )?;
319        let rows = stmt.query_map([], |r| {
320            Ok((
321                r.get::<_, i64>(0)?,
322                r.get::<_, i64>(1)?,
323                r.get::<_, String>(2)?,
324                r.get::<_, String>(3)?,
325                r.get::<_, Option<i64>>(4)?,
326            ))
327        })?;
328        for row in rows {
329            let (id, file_id, path, imported_path, old) = row?;
330            if skip.contains(&file_id) {
331                continue;
332            }
333            // Resolution reads only the imported path (the stored type is already
334            // the reclassified one, and External/Stdlib rows are skipped above).
335            let import = ImportInfo {
336                imported_path,
337                import_type: crate::models::ImportType::Internal,
338                line_number: 0,
339                imported_symbols: None,
340            };
341            let new = ctx.resolve_import(&path, &import, resolver);
342            if new != old {
343                changed.push((id, new));
344            }
345        }
346    }
347    let mut stmt = tx.prepare("UPDATE file_dependencies SET resolved_file_id = ? WHERE id = ?")?;
348    for (id, new) in &changed {
349        stmt.execute(rusqlite::params![new, id])?;
350    }
351    let mut total = changed.len();
352
353    let mut changed: Vec<(i64, Option<i64>)> = Vec::new();
354    {
355        let mut stmt = tx.prepare(
356            "SELECT e.id, e.file_id, f.path, e.source_path, e.resolved_source_id
357             FROM file_exports e JOIN files f ON f.id = e.file_id",
358        )?;
359        let rows = stmt.query_map([], |r| {
360            Ok((
361                r.get::<_, i64>(0)?,
362                r.get::<_, i64>(1)?,
363                r.get::<_, String>(2)?,
364                r.get::<_, String>(3)?,
365                r.get::<_, Option<i64>>(4)?,
366            ))
367        })?;
368        for row in rows {
369            let (id, file_id, path, source_path, old) = row?;
370            if skip.contains(&file_id) {
371                continue;
372            }
373            let export = ExportInfo {
374                exported_symbol: None,
375                source_path,
376                line_number: 0,
377            };
378            let new = ctx.resolve_export(&path, &export, resolver);
379            if new != old {
380                changed.push((id, new));
381            }
382        }
383    }
384    let mut stmt = tx.prepare("UPDATE file_exports SET resolved_source_id = ? WHERE id = ?")?;
385    for (id, new) in &changed {
386        stmt.execute(rusqlite::params![new, id])?;
387    }
388    total += changed.len();
389    Ok(total)
390}
391
392/// What import extraction yields for one file: its imports, its re-exports, and
393/// the `(package key, member)` rows its content declares (`package_members`).
394type Extracted = (Vec<ImportInfo>, Vec<ExportInfo>, Vec<(String, String)>);
395
396/// Imports, re-exports and declared package memberships of one file, by
397/// language. `path_str` is the path as walked (for the nearest tsconfig).
398fn extract_imports(
399    language: Language,
400    content: &str,
401    path_str: &str,
402    root: &Path,
403    tsconfigs: &HashMap<PathBuf, crate::parsers::tsconfig::PathAliasMap>,
404) -> Extracted {
405    // Extract dependencies and exports for supported languages
406    let mut parsed_exports: Vec<ExportInfo> = Vec::new();
407    let mut declared_namespaces: Vec<(String, String)> = Vec::new();
408    let dependencies = match language {
409        Language::Rust => match RustDependencyExtractor::extract_dependencies(content) {
410            Ok(deps) => deps,
411            Err(e) => {
412                log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
413                Vec::new()
414            }
415        },
416        Language::Python => match PythonDependencyExtractor::extract_dependencies(content) {
417            Ok(deps) => deps,
418            Err(e) => {
419                log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
420                Vec::new()
421            }
422        },
423        // A Qt Linguist translation file (XML) shares the `.ts` extension
424        Language::TypeScript if crate::parsers::is_qt_linguist_ts(content) => Vec::new(),
425        Language::TypeScript | Language::JavaScript => {
426            // Find nearest tsconfig for path alias resolution. One parse
427            // yields both the imports and the re-exports.
428            let alias_map =
429                crate::dependency_resolve::find_nearest_tsconfig(path_str, root, tsconfigs);
430            match TypeScriptDependencyExtractor::extract_dependencies_and_exports(
431                content, alias_map,
432            ) {
433                Ok((deps, exports)) => {
434                    parsed_exports = exports;
435                    deps
436                }
437                Err(e) => {
438                    log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
439                    Vec::new()
440                }
441            }
442        }
443        Language::Go => match GoDependencyExtractor::extract_dependencies(content) {
444            Ok(deps) => deps,
445            Err(e) => {
446                log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
447                Vec::new()
448            }
449        },
450        Language::Java => match JavaDependencyExtractor::extract_dependencies(content) {
451            Ok(deps) => deps,
452            Err(e) => {
453                log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
454                Vec::new()
455            }
456        },
457        Language::C => match CDependencyExtractor::extract_dependencies(content) {
458            Ok(deps) => deps,
459            Err(e) => {
460                log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
461                Vec::new()
462            }
463        },
464        Language::Cpp => match CppDependencyExtractor::extract_dependencies(content) {
465            Ok(deps) => deps,
466            Err(e) => {
467                log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
468                Vec::new()
469            }
470        },
471        Language::CSharp => {
472            match CSharpDependencyExtractor::extract_dependencies_and_members(content) {
473                Ok((deps, members)) => {
474                    declared_namespaces = members;
475                    deps
476                }
477                Err(e) => {
478                    log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
479                    Vec::new()
480                }
481            }
482        }
483        Language::PHP => match PhpDependencyExtractor::extract_dependencies(content) {
484            Ok(deps) => deps,
485            Err(e) => {
486                log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
487                Vec::new()
488            }
489        },
490        Language::Ruby => match RubyDependencyExtractor::extract_dependencies(content) {
491            Ok(deps) => deps,
492            Err(e) => {
493                log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
494                Vec::new()
495            }
496        },
497        Language::Kotlin => match KotlinDependencyExtractor::extract_dependencies(content) {
498            Ok(deps) => deps,
499            Err(e) => {
500                log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
501                Vec::new()
502            }
503        },
504        Language::Zig => match ZigDependencyExtractor::extract_dependencies(content) {
505            Ok(deps) => deps,
506            Err(e) => {
507                log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
508                Vec::new()
509            }
510        },
511        // Read from the text: Swift has no tree-sitter grammar in this build
512        Language::Swift => match SwiftDependencyExtractor::extract_dependencies(content) {
513            Ok(deps) => deps,
514            Err(e) => {
515                log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
516                Vec::new()
517            }
518        },
519        Language::Vue => {
520            // Find nearest tsconfig for path alias resolution. One parse
521            // per script block yields both the imports and the re-exports.
522            let alias_map =
523                crate::dependency_resolve::find_nearest_tsconfig(path_str, root, tsconfigs);
524            match VueDependencyExtractor::extract_dependencies_and_exports(content, alias_map) {
525                Ok((deps, exports)) => {
526                    parsed_exports = exports;
527                    deps
528                }
529                Err(e) => {
530                    log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
531                    Vec::new()
532                }
533            }
534        }
535        Language::Svelte => match SvelteDependencyExtractor::extract_dependencies(content) {
536            Ok(deps) => deps,
537            Err(e) => {
538                log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
539                Vec::new()
540            }
541        },
542        // Other languages not yet implemented
543        _ => Vec::new(),
544    };
545
546    // Exports (barrel re-export tracking) came out of the same parse as the
547    // dependencies above; only TypeScript/JavaScript/Vue have them.
548    let members = match language {
549        Language::Java | Language::Kotlin => {
550            crate::parsers::java::jvm_package_members(path_str, content)
551        }
552        Language::CSharp => declared_namespaces,
553        _ => Vec::new(),
554    };
555    (dependencies, parsed_exports, members)
556}
557
558/// Manages the indexing process
559pub struct Indexer {
560    cache: CacheManager,
561    config: IndexConfig,
562    /// `(max_files, max_bytes)` per batch; `None` = defaults / env overrides.
563    batch_limits: Option<(usize, u64)>,
564    /// `(max_files, max_bytes)` of the delta before it is merged into a new base;
565    /// `None` = [`DELTA_MAX_FILES`] and [`DELTA_MAX_CORPUS_PERCENT`] of the corpus.
566    merge_limits: Option<(usize, u64)>,
567    /// `(max_files, max_bytes)` of the recent segment before it is folded into the
568    /// delta; `None` = [`RECENT_MAX_FILES`] and the merge byte limit over
569    /// [`RECENT_BYTES_DIVISOR`].
570    recent_limits: Option<(usize, u64)>,
571    /// Crash tests: the write point at which the process aborts.
572    abort_at: Option<String>,
573    /// The rule-file record the current `index` run saw, when it differs from
574    /// the stored one: written after the run succeeds.
575    rules_pending: std::sync::Mutex<Option<String>>,
576}
577
578/// The path resolver of this process's last delta publish: `(meta.db, publish id,
579/// resolver)`. The next publish over that manifest patches it with its added and
580/// deleted paths instead of loading every path; any other publish (another process,
581/// a full build, a cleared cache) has another publish id, and the next one loads
582/// again.
583static RESOLVER_CACHE: Mutex<Option<(PathBuf, u64, crate::dependency::PathResolver)>> =
584    Mutex::new(None);
585
586/// Files the recent segment may hold before an update folds it into the delta.
587pub const RECENT_MAX_FILES: usize = 256;
588/// The recent segment's text limit: the delta's (merge) limit divided by this.
589pub const RECENT_BYTES_DIVISOR: u64 = 16;
590
591/// Exit code of a process stopped by `Indexer::set_abort_point`.
592#[doc(hidden)]
593pub const ABORT_EXIT_CODE: i32 = 86;
594
595/// Files the delta may hold before an update merges it into a new base.
596pub const DELTA_MAX_FILES: usize = 2000;
597/// Text bytes the delta may hold, as a percentage of the live corpus, before an
598/// update merges it into a new base.
599pub const DELTA_MAX_CORPUS_PERCENT: u64 = 5;
600
601/// Default cap on files per batch.
602pub const BATCH_MAX_FILES: usize = 5000;
603/// Default cap on bytes of text per batch. Bounds the per-batch memory of the
604/// read pool (file contents) and of the trigram build (postings), whatever the
605/// file count: a 9,000-file tree of large files no longer builds one giant
606/// in-memory index.
607pub const BATCH_MAX_BYTES: u64 = 48 << 20;
608
609/// Cut `sizes` (in file order) into consecutive batches of at most `max_files`
610/// files and, past the first file of a batch, at most `max_bytes` bytes.
611pub fn plan_batches(
612    sizes: &[u64],
613    max_files: usize,
614    max_bytes: u64,
615) -> Vec<std::ops::Range<usize>> {
616    let max_files = max_files.max(1);
617    let mut batches = Vec::new();
618    let mut start = 0usize;
619    let mut bytes = 0u64;
620    for (i, &size) in sizes.iter().enumerate() {
621        let count = i - start;
622        if count > 0 && (count >= max_files || bytes.saturating_add(size) > max_bytes) {
623            batches.push(start..i);
624            start = i;
625            bytes = 0;
626        }
627        bytes = bytes.saturating_add(size);
628    }
629    if start < sizes.len() {
630        batches.push(start..sizes.len());
631    }
632    batches
633}
634
635/// The `[index] include.patterns` / `exclude.patterns` policy, compiled once.
636///
637/// Built on `ignore::overrides::Override`, so the patterns follow gitignore rules
638/// exactly as the walker applies them: a pattern containing `/` is anchored at the
639/// workspace root, a bare name matches at any depth, `*` does not cross `/`.
640/// Includes are whitelist globs, excludes are `!`-prefixed. With only includes,
641/// non-matching *files* are dropped but directories are still walked, so
642/// `include = ["src/**/*.rs"]` works without listing `src/`.
643///
644/// The same policy must answer "would Reflex index this path?" everywhere: the
645/// walker, the working-tree freshness check and the watcher. If they disagree, an
646/// edit to an excluded file marks the index permanently stale.
647#[derive(Clone, Debug)]
648pub struct PathPolicy {
649    overrides: Option<ignore::overrides::Override>,
650    /// `[index] text_tier`.
651    text_tier: bool,
652    /// `[index] languages`; empty = every supported language.
653    languages: Vec<Language>,
654    /// `[index] mode`.
655    mode: IndexMode,
656    /// `[index] hidden`.
657    hidden: bool,
658}
659
660impl Default for PathPolicy {
661    fn default() -> Self {
662        Self {
663            overrides: None,
664            text_tier: true,
665            languages: Vec::new(),
666            mode: IndexMode::Tracked,
667            hidden: false,
668        }
669    }
670}
671
672impl PathPolicy {
673    /// Compile the policy from an index config. No patterns → a policy that admits
674    /// every path the language rules allow. An invalid pattern is logged and skipped.
675    pub fn from_config(root: &Path, config: &IndexConfig) -> Self {
676        let mut policy = Self {
677            overrides: None,
678            text_tier: config.text_tier,
679            languages: config.languages.clone(),
680            mode: config.mode,
681            hidden: config.hidden,
682        };
683        if config.include_patterns.is_empty() && config.exclude_patterns.is_empty() {
684            return policy;
685        }
686        let mut builder = ignore::overrides::OverrideBuilder::new(root);
687        for pat in &config.include_patterns {
688            if let Err(e) = builder.add(pat) {
689                log::warn!("Invalid [index] include pattern '{}': {}", pat, e);
690            }
691        }
692        for pat in &config.exclude_patterns {
693            let negated = format!("!{}", pat.trim_start_matches('!'));
694            if let Err(e) = builder.add(&negated) {
695                log::warn!("Invalid [index] exclude pattern '{}': {}", pat, e);
696            }
697        }
698        match builder.build() {
699            Ok(ov) => policy.overrides = Some(ov),
700            Err(e) => log::warn!("Failed to build [index] include/exclude policy: {}", e),
701        }
702        policy
703    }
704
705    /// The language a file at `path` would be indexed as, or `None` when the
706    /// policy would not index it (excluded by pattern, tier off, parser not
707    /// wanted). Path only: no filesystem access, so it is the same answer for the
708    /// walker, the freshness check and the watcher.
709    pub fn classify(&self, path: &Path) -> Option<Language> {
710        if !self.admits(path, false) {
711            return None;
712        }
713        let lang = Language::from_path(path);
714        let name = path.file_name().and_then(|n| n.to_str()).unwrap_or("");
715
716        match lang {
717            // Indexed only in tracked mode, and excluded from searches by default.
718            Language::Lock | Language::Generated => {
719                (self.mode == IndexMode::Tracked).then_some(lang)
720            }
721            // The plain-text tier: docs, config, templates and, in tracked mode,
722            // every other non-binary file. Deliberately NOT subject to `languages`.
723            // That option means "which PARSERS do I care about"; a user with
724            // languages = ["rust"] would otherwise lose the text tier silently,
725            // which is the very bug this tier exists to fix. `text_tier = false`
726            // is the way to turn it off.
727            Language::Text => {
728                if !self.text_tier {
729                    return None;
730                }
731                match self.mode {
732                    IndexMode::Tracked => Some(lang),
733                    IndexMode::Allowlist => crate::models::is_text_tier_file(name).then_some(lang),
734                }
735            }
736            Language::Unknown => None,
737            // Code without a working grammar (Swift): in tracked mode it is still a
738            // text file an agent greps, so it is indexed; symbol queries skip it.
739            code if !code.is_supported() => {
740                (self.mode == IndexMode::Tracked && self.text_tier).then_some(code)
741            }
742            code => {
743                if !self.languages.is_empty() && !self.languages.contains(&code) {
744                    log::debug!(
745                        "Skipping {} ({:?} not in configured languages)",
746                        path.display(),
747                        code
748                    );
749                    return None;
750                }
751                Some(code)
752            }
753        }
754    }
755
756    /// Whether a workspace-RELATIVE path is under a directory the walker would
757    /// descend into. With `hidden = false` (the default, like ripgrep) no dot
758    /// segment is; with `hidden = true` only `.git/` and `.reflex/` are skipped.
759    ///
760    /// Relative paths only: an absolute path's own ancestors (`/tmp/.cache/…`) are
761    /// none of the walker's business.
762    pub fn hidden_ok(&self, rel: &Path) -> bool {
763        rel.components()
764            .filter_map(|c| c.as_os_str().to_str())
765            .all(|seg| {
766                if seg == "." || seg == ".." {
767                    return true;
768                }
769                if seg == ".git" || seg == crate::cache::CACHE_DIR {
770                    return false;
771                }
772                self.hidden || !is_hidden_segment(seg)
773            })
774    }
775
776    /// `[index] hidden`.
777    pub fn hidden(&self) -> bool {
778        self.hidden
779    }
780
781    /// Whether the policy admits this path. Directories are always admitted (the
782    /// walker descends; files decide), unless an exclude names them.
783    pub fn admits(&self, path: &Path, is_dir: bool) -> bool {
784        match &self.overrides {
785            None => true,
786            Some(ov) => !ov.matched(path, is_dir).is_ignore(),
787        }
788    }
789
790    /// The walker-side view of the policy.
791    pub fn overrides(&self) -> Option<&ignore::overrides::Override> {
792        self.overrides.as_ref()
793    }
794}
795
796/// A walk before the binary check (see `Indexer::walk_candidates`).
797#[derive(Debug, Default)]
798struct Walked {
799    found: Discovered,
800    /// Per entry of `found`: whether its type is not code (so it is sniffed).
801    non_code: Vec<bool>,
802}
803
804/// What one directory walk found.
805#[derive(Debug, Default)]
806struct Discovered {
807    /// Each file relative to the root, with forward slashes: the path the stores
808    /// and meta.db use. The walker's own path is `root.join(rel)`.
809    rels: Vec<String>,
810    /// On-disk size of each entry of `files` (0 when unknown); drives batching.
811    sizes: Vec<u64>,
812    /// The `stat` of each entry of `files`, taken during the walk.
813    metas: Vec<Option<crate::cache::FileStat>>,
814    skipped_too_large: usize,
815    skipped_bytes_too_large: u64,
816    skipped_binary: usize,
817}
818
819/// The paths a library update names, relative to the root (`/`-separated): the
820/// walk goes down through their ancestors and into them, and nowhere else.
821#[derive(Debug, Default)]
822struct Targets {
823    /// Each named path (a file or a directory).
824    exact: std::collections::HashSet<String>,
825    /// Every proper ancestor directory of a named path.
826    ancestors: std::collections::HashSet<String>,
827}
828
829impl Targets {
830    fn new(paths: &[String]) -> Self {
831        let mut out = Self::default();
832        for p in paths {
833            let mut end = p.len();
834            while let Some(slash) = p[..end].rfind('/') {
835                out.ancestors.insert(p[..slash].to_string());
836                end = slash;
837            }
838            out.exact.insert(p.clone());
839        }
840        out
841    }
842
843    /// Whether `rel` is a named path or lies under one.
844    fn covers(&self, rel: &str) -> bool {
845        if self.exact.contains(rel) {
846            return true;
847        }
848        let mut end = rel.len();
849        while let Some(slash) = rel[..end].rfind('/') {
850            if self.exact.contains(&rel[..slash]) {
851                return true;
852            }
853            end = slash;
854        }
855        false
856    }
857
858    /// Whether the walk enters `rel`: the root, an ancestor of a named path, or a
859    /// path a named path covers.
860    fn admits(&self, rel: &str) -> bool {
861        rel.is_empty() || self.ancestors.contains(rel) || self.covers(rel)
862    }
863}
864
865/// Walk positions for the files a library update rewrites (`rewrites`, in walk
866/// order; each with its row when it has one), found by probing `files` in
867/// `walk_seq` order with the walk-order comparator. A rewritten file keeps its
868/// position while its neighbours still bracket it; a new or moved one gets a
869/// value between the rows that now bracket it. Rows in `skip` (deleted) are left
870/// out. `None` when a comparison cannot be made or a gap has no room: the caller
871/// then runs a full index, which renumbers.
872fn place_rewrites(
873    conn: &rusqlite::Connection,
874    walk: &mut crate::walk_order::WalkOrder<'_>,
875    rewrites: &[(&str, Option<&crate::meta_update::StoredFile>)],
876    skip: &std::collections::HashSet<i64>,
877) -> Result<Option<HashMap<String, i64>>> {
878    use crate::meta_update::row_near_seq;
879    use std::cmp::Ordering::{Greater, Less};
880    let mut skip = skip.clone();
881    let mut kept: HashMap<&str, i64> = HashMap::new();
882    // Rows whose neighbours no longer bracket them are placed anew; a row moved
883    // out can change another's neighbours, so check again until nothing moves.
884    let mut moved: std::collections::HashSet<&str> = std::collections::HashSet::new();
885    for _ in 0..4 {
886        kept.clear();
887        let mut changed = false;
888        for (path, row) in rewrites {
889            let Some(row) = row else { continue };
890            if moved.contains(path) {
891                continue;
892            }
893            let before = row_near_seq(conn, row.walk_seq, true, &skip)?;
894            let after = row_near_seq(conn, row.walk_seq + 1, false, &skip)?;
895            let fits = before
896                .as_ref()
897                .is_none_or(|(_, p, _)| walk.cmp(p, path) == Some(Less))
898                && after
899                    .as_ref()
900                    .is_none_or(|(_, p, _)| walk.cmp(path, p) == Some(Less));
901            if fits {
902                kept.insert(path, row.walk_seq);
903            } else {
904                moved.insert(path);
905                skip.insert(row.id);
906                changed = true;
907            }
908        }
909        if !changed {
910            break;
911        }
912    }
913
914    let bounds: (Option<i64>, Option<i64>) =
915        conn.query_row("SELECT MIN(walk_seq), MAX(walk_seq) FROM files", [], |r| {
916            Ok((r.get(0)?, r.get(1)?))
917        })?;
918    // Each file to place, with the positions of the rows that bracket it.
919    let mut gaps: Vec<(&str, Option<i64>, Option<i64>)> = Vec::new();
920    for (path, row) in rewrites {
921        if row.is_some() && !moved.contains(path) {
922            continue;
923        }
924        let (before, after) = match bounds {
925            (Some(min), Some(max)) => {
926                // The smallest position whose first row comes after `path`.
927                let (mut lo, mut hi) = (min, max + 1);
928                while lo < hi {
929                    let mid = lo + (hi - lo) / 2;
930                    let after_path = match row_near_seq(conn, mid, false, &skip)? {
931                        None => true,
932                        Some((_, q, _)) => match walk.cmp(&q, path) {
933                            Some(Greater) => true,
934                            Some(Less) => false,
935                            _ => return Ok(None),
936                        },
937                    };
938                    if after_path {
939                        hi = mid;
940                    } else {
941                        lo = mid + 1;
942                    }
943                }
944                (
945                    row_near_seq(conn, lo, true, &skip)?.map(|r| r.2),
946                    row_near_seq(conn, lo, false, &skip)?.map(|r| r.2),
947                )
948            }
949            _ => (None, None),
950        };
951        gaps.push((path, before, after));
952    }
953    let mut out: HashMap<String, i64> = kept
954        .into_iter()
955        .map(|(p, seq)| (p.to_string(), seq))
956        .collect();
957    // Files that share a gap are spread over it, in walk order.
958    let mut i = 0;
959    while i < gaps.len() {
960        let (_, before, after) = gaps[i];
961        let mut j = i;
962        while j < gaps.len() && (gaps[j].1, gaps[j].2) == (before, after) {
963            j += 1;
964        }
965        let k = (j - i) as i64;
966        for (n, (path, _, _)) in gaps[i..j].iter().enumerate() {
967            let n = n as i64;
968            let seq = match (before, after) {
969                (Some(a), Some(b)) => {
970                    let step = (b - a) / (k + 1);
971                    if step < 1 {
972                        return Ok(None);
973                    }
974                    a.checked_add(step * (n + 1))
975                }
976                (Some(a), None) => a.checked_add(WALK_SEQ_GAP * (n + 1)),
977                (None, Some(b)) => b.checked_sub(WALK_SEQ_GAP * (k - n)),
978                (None, None) => Some(WALK_SEQ_GAP * n),
979            };
980            let Some(seq) = seq else { return Ok(None) };
981            out.insert(path.to_string(), seq);
982        }
983        i = j;
984    }
985    Ok(Some(out))
986}
987
988/// `file_path` relative to `root` with forward slashes (a path that is not under
989/// `root` loses a leading `./`): the path the stores and meta.db use, the same
990/// on every OS.
991fn normalize_rel(root: &Path, file_path: &Path) -> String {
992    match file_path.strip_prefix(root) {
993        Ok(rel) => rel.to_string_lossy().replace('\\', "/"),
994        Err(_) => file_path
995            .to_string_lossy()
996            .trim_start_matches("./")
997            .replace('\\', "/"),
998    }
999}
1000
1001/// A path segment the walker treats as hidden: a dot-name other than `.` / `..`.
1002/// Shared by the walker policy, the freshness check and the zero-result hint.
1003pub fn is_hidden_segment(seg: &str) -> bool {
1004    seg.len() > 1 && seg.starts_with('.') && seg != ".."
1005}
1006
1007/// ripgrep's binary rule: a NUL byte anywhere in the file.
1008///
1009/// Not "in the first 8 KB": a protobuf blob on the Kubernetes checkout
1010/// (`swagger.pb`, 4109 word matches) carries its first NUL past that point, and
1011/// ripgrep still skips it. The indexer holds the whole file in memory when it
1012/// asks, so the full scan is free; `memchr` makes it a few GB/s.
1013pub fn is_binary(bytes: &[u8]) -> bool {
1014    memchr::memchr(0, bytes).is_some()
1015}
1016
1017/// [`is_binary`] on the file at `path`. A file that cannot be read is not called
1018/// binary here; the read that follows reports the error. Callers apply this only
1019/// to non-code files under `max_file_size`.
1020pub fn looks_binary(path: &Path) -> bool {
1021    match std::fs::read(path) {
1022        Ok(bytes) => is_binary(&bytes),
1023        Err(_) => false,
1024    }
1025}
1026
1027/// Phase timings of one run, logged as one line.
1028struct Laps {
1029    last: Instant,
1030    line: String,
1031}
1032
1033impl Laps {
1034    fn new() -> Self {
1035        Self {
1036            last: Instant::now(),
1037            line: String::new(),
1038        }
1039    }
1040
1041    fn lap(&mut self, name: &str) {
1042        use std::fmt::Write;
1043        let _ = write!(
1044            self.line,
1045            " {}={:.1}",
1046            name,
1047            self.last.elapsed().as_secs_f64() * 1000.0
1048        );
1049        self.last = Instant::now();
1050    }
1051}
1052
1053/// What `Indexer::update_paths` does with one named path.
1054enum Named {
1055    /// Nothing indexed can depend on it (outside the tree, `.git/`, `.reflex/`).
1056    Skip,
1057    /// It needs a full run: an ignore file, the project config, a resolver config,
1058    /// the root itself, or a path that leaves the root.
1059    Full,
1060    /// A path under the root, relative and `/`-separated.
1061    Path(String),
1062}
1063
1064fn classify_named(root: &Path, abs_root: &Path, path: &Path) -> Named {
1065    let rel = if path.is_absolute() {
1066        match path
1067            .strip_prefix(root)
1068            .or_else(|_| path.strip_prefix(abs_root))
1069        {
1070            Ok(rel) => rel,
1071            Err(_) => return Named::Skip,
1072        }
1073    } else {
1074        path
1075    };
1076    let mut parts: Vec<String> = Vec::new();
1077    for component in rel.components() {
1078        match component {
1079            std::path::Component::Normal(s) => parts.push(s.to_string_lossy().into_owned()),
1080            std::path::Component::CurDir => {}
1081            _ => return Named::Full,
1082        }
1083    }
1084    let Some(name) = parts.last() else {
1085        return Named::Full;
1086    };
1087    if parts[0] == ".git" {
1088        return Named::Skip;
1089    }
1090    if parts[0] == crate::cache::CACHE_DIR {
1091        return if parts.len() == 2 && parts[1] == "config.toml" {
1092            Named::Full
1093        } else {
1094            Named::Skip
1095        };
1096    }
1097    if is_ignore_file_name(name) || crate::dependency_resolve::is_resolver_config_name(name) {
1098        return Named::Full;
1099    }
1100    Named::Path(parts.join("/"))
1101}
1102
1103/// `statistics` key: the rule files an index run saw, as a JSON map from path
1104/// (relative to the root) to the blake3 of its bytes (`""` when absent). The rule
1105/// files decide WHICH files are indexed, so a change to one makes the index stale
1106/// although no indexed file changed. Older binaries ignore the key.
1107pub(crate) const RULE_FILES_KEY: &str = "rule_files";
1108
1109/// Rule files compared on every freshness check: Reflex's config and the root's
1110/// ignore files. Nested ignore files are compared when git lists them.
1111pub(crate) const FIXED_RULE_FILES: [&str; 4] =
1112    [".reflex/config.toml", ".gitignore", ".ignore", ".rgignore"];
1113
1114/// Whether `name` is an ignore file the walker reads in every directory.
1115pub(crate) fn is_ignore_file_name(name: &str) -> bool {
1116    matches!(name, ".gitignore" | ".ignore" | ".rgignore")
1117}
1118
1119/// Whether the path `rel` (relative, `/`-separated) is a rule file.
1120pub(crate) fn is_rule_file(rel: &str) -> bool {
1121    rel == FIXED_RULE_FILES[0] || is_ignore_file_name(rel.rsplit('/').next().unwrap_or(rel))
1122}
1123
1124/// The blake3 of the rule file at `rel`, or `""` when it cannot be read.
1125pub(crate) fn rule_file_hash(root: &Path, rel: &str) -> String {
1126    match std::fs::read(root.join(rel)) {
1127        Ok(bytes) => blake3::hash(&bytes).to_hex().to_string(),
1128        Err(_) => String::new(),
1129    }
1130}
1131
1132/// The rule-file record of an index run over `root`: the fixed files, and every
1133/// ignore file in `dirty` (the paths `git status` listed).
1134fn rule_files_record<'a>(root: &Path, dirty: impl IntoIterator<Item = &'a String>) -> String {
1135    let mut record: std::collections::BTreeMap<String, String> = FIXED_RULE_FILES
1136        .iter()
1137        .map(|rel| (rel.to_string(), rule_file_hash(root, rel)))
1138        .collect();
1139    for rel in dirty {
1140        if is_rule_file(rel) {
1141            record.insert(rel.clone(), rule_file_hash(root, rel));
1142        }
1143    }
1144    serde_json::to_string(&record).unwrap_or_default()
1145}
1146
1147/// Whether a directory holds a file named like a resolver config at any depth (no
1148/// ignore rules: a superset of what the config walk would find).
1149fn holds_resolver_config(dir: &Path) -> bool {
1150    ignore::WalkBuilder::new(dir)
1151        .standard_filters(false)
1152        .build()
1153        .flatten()
1154        .any(|e| {
1155            e.file_name()
1156                .to_str()
1157                .is_some_and(crate::dependency_resolve::is_resolver_config_name)
1158        })
1159}
1160
1161/// Drops the shared query handles for a workspace when an index run ends,
1162/// on every exit path including errors and panics.
1163struct InvalidateOnDrop(std::path::PathBuf);
1164
1165impl Drop for InvalidateOnDrop {
1166    fn drop(&mut self) {
1167        crate::query::invalidate_caches(&self.0);
1168    }
1169}
1170
1171impl Indexer {
1172    /// Create a new indexer with the given cache manager and config
1173    pub fn new(cache: CacheManager, config: IndexConfig) -> Self {
1174        Self {
1175            cache,
1176            config,
1177            batch_limits: None,
1178            merge_limits: None,
1179            recent_limits: None,
1180            abort_at: None,
1181            rules_pending: std::sync::Mutex::new(None),
1182        }
1183    }
1184
1185    /// Override the per-batch limits (files, bytes). For tests that need to
1186    /// exercise multi-batch builds on small trees; production reads
1187    /// `REFLEX_INDEX_BATCH_FILES` / `REFLEX_INDEX_BATCH_BYTES` or the defaults.
1188    #[doc(hidden)]
1189    pub fn set_batch_limits(&mut self, max_files: usize, max_bytes: u64) {
1190        self.batch_limits = Some((max_files, max_bytes));
1191    }
1192
1193    /// Override the delta's merge limits (files, text bytes). For tests: a small
1194    /// tree would otherwise merge on every update under the 5 % rule.
1195    #[doc(hidden)]
1196    pub fn set_merge_limits(&mut self, max_files: usize, max_bytes: u64) {
1197        self.merge_limits = Some((max_files, max_bytes));
1198    }
1199
1200    /// Abort the process at a named write point, to test crash safety. A delta
1201    /// publish has `delta-files` (stores written), `unlinked` (fixed names
1202    /// removed), `manifest` (published) and `meta` (meta.db committed); a full
1203    /// build has `base-files`, `base-manifest` and `base-meta`.
1204    #[doc(hidden)]
1205    pub fn set_abort_point(&mut self, point: &str) {
1206        self.abort_at = Some(point.to_string());
1207    }
1208
1209    fn abort_point(&self, point: &str) {
1210        if self.abort_at.as_deref() == Some(point) {
1211            // No destructor, no later write: what a killed process leaves.
1212            std::process::exit(ABORT_EXIT_CODE);
1213        }
1214    }
1215
1216    /// Override the recent segment's limits (files, text bytes) before it is
1217    /// folded into the delta. For tests.
1218    #[doc(hidden)]
1219    pub fn set_recent_limits(&mut self, max_files: usize, max_bytes: u64) {
1220        self.recent_limits = Some((max_files, max_bytes));
1221    }
1222
1223    fn recent_limits(&self, max_files: usize, max_bytes: u64) -> (usize, u64) {
1224        self.recent_limits.unwrap_or((
1225            RECENT_MAX_FILES.min(max_files),
1226            max_bytes / RECENT_BYTES_DIVISOR,
1227        ))
1228    }
1229
1230    fn merge_limits(&self, live_corpus_bytes: u64) -> (usize, u64) {
1231        self.merge_limits.unwrap_or((
1232            DELTA_MAX_FILES,
1233            live_corpus_bytes * DELTA_MAX_CORPUS_PERCENT / 100,
1234        ))
1235    }
1236
1237    fn batch_limits(&self) -> (usize, u64) {
1238        if let Some(limits) = self.batch_limits {
1239            return limits;
1240        }
1241        let files = std::env::var("REFLEX_INDEX_BATCH_FILES")
1242            .ok()
1243            .and_then(|v| v.parse::<usize>().ok())
1244            .filter(|&n| n > 0)
1245            .unwrap_or(BATCH_MAX_FILES);
1246        let bytes = std::env::var("REFLEX_INDEX_BATCH_BYTES")
1247            .ok()
1248            .and_then(|v| v.parse::<u64>().ok())
1249            .filter(|&n| n > 0)
1250            .unwrap_or(BATCH_MAX_BYTES);
1251        (files, bytes)
1252    }
1253
1254    /// The `[index] include/exclude` policy for a workspace root.
1255    pub fn path_policy(&self, root: &Path) -> PathPolicy {
1256        PathPolicy::from_config(root, &self.config)
1257    }
1258
1259    /// Build or update the index for the given root directory
1260    pub fn index(&self, root: impl AsRef<Path>, show_progress: bool) -> Result<IndexStats> {
1261        self.index_with_callback(root, show_progress, None)
1262    }
1263
1264    /// Bring the index up to date with changes to `paths` alone, without walking
1265    /// the tree: for a caller that already knows what changed (a file watcher, an
1266    /// editor). `paths` are files or directories, absolute or relative to `root`,
1267    /// that were added, modified or deleted; a rename names both paths.
1268    ///
1269    /// When every changed path is named, the index then holds what
1270    /// [`Indexer::index`] would build. The run falls back to [`Indexer::index`]
1271    /// when the change cannot be applied alone: an ignore file, `.reflex/config.toml`
1272    /// or a resolver config (`go.mod`, `tsconfig.json`, ...) is involved, the branch
1273    /// changed, the delta would pass its merge limits, or the index is not one an
1274    /// update can start from.
1275    pub fn update_paths(&self, root: impl AsRef<Path>, paths: &[PathBuf]) -> Result<IndexStats> {
1276        let root = root.as_ref();
1277        match self.try_update_paths(root, paths)? {
1278            Some(stats) => Ok(stats),
1279            None => {
1280                log::info!("update_paths: the change needs a full index run");
1281                self.index(root, false)
1282            }
1283        }
1284    }
1285
1286    /// [`Self::update_paths`] without the fallback: `None` when a full run is
1287    /// needed (nothing was written). For tests that check which path ran.
1288    #[doc(hidden)]
1289    pub fn try_update_paths(&self, root: &Path, paths: &[PathBuf]) -> Result<Option<IndexStats>> {
1290        let started = Instant::now();
1291        let mut laps = Laps::new();
1292        let cache_dir = self.cache.path().to_path_buf();
1293        if !cache_dir.join(crate::cache::META_DB).exists() {
1294            return Ok(None);
1295        }
1296        let abs_root = root.canonicalize().unwrap_or_else(|_| root.to_path_buf());
1297        let mut named: Vec<String> = Vec::new();
1298        for path in paths {
1299            match classify_named(root, &abs_root, path) {
1300                Named::Skip => {}
1301                Named::Full => return Ok(None),
1302                Named::Path(rel) => named.push(rel),
1303            }
1304        }
1305        named.sort();
1306        named.dedup();
1307
1308        // `git status` of the named paths runs while the rest is prepared.
1309        let in_git = crate::git::is_git_repo(root);
1310        let spawn_status = |specs: Vec<String>| {
1311            let root = root.to_path_buf();
1312            std::thread::spawn(move || {
1313                if specs.len() > 1000 {
1314                    crate::git::changed_paths(&root)
1315                } else {
1316                    let refs: Vec<&str> = specs.iter().map(String::as_str).collect();
1317                    crate::git::changed_paths_in(&root, &refs)
1318                }
1319            })
1320        };
1321        let mut status_jobs = Vec::new();
1322        if in_git && !named.is_empty() {
1323            status_jobs.push(spawn_status(named.clone()));
1324        }
1325
1326        // The start of a run, as in `index_with_callback`.
1327        let _index_lock = crate::atomic_write::IndexLock::acquire_with_timeout(
1328            &cache_dir,
1329            std::time::Duration::from_secs(self.config.lock_wait_secs),
1330        )?;
1331        crate::atomic_write::remove_stale_tmp(&cache_dir);
1332        crate::query::invalidate_caches(root);
1333        let _invalidate_on_exit = InvalidateOnDrop(root.to_path_buf());
1334        self.yield_to_symbol_pass(&cache_dir)?;
1335        laps.lap("lock");
1336        self.cache.assert_writable(false)?;
1337        if !self.cache.check_schema_hash().unwrap_or(false)
1338            || !self.cache.check_extraction_hash().unwrap_or(false)
1339        {
1340            return Ok(None);
1341        }
1342        let run_start = self.run_marker(&cache_dir);
1343        laps.lap("checks");
1344
1345        let conn = crate::cache::open_meta_db(cache_dir.join(crate::cache::META_DB))?;
1346        let file_count = crate::meta_update::count_files(&conn)?;
1347        let stored_digest = crate::meta_update::get_statistic(&conn, RESOLVER_DIGEST_KEY)?;
1348        let meta_generation = crate::meta_update::get_statistic(&conn, INDEX_GENERATION_KEY)?
1349            .and_then(|g| g.parse::<u64>().ok());
1350        let synced_branch =
1351            crate::meta_update::get_statistic(&conn, crate::meta_update::SYNCED_BRANCH_KEY)?;
1352        let (Some(generation), Ok(indexed)) =
1353            (meta_generation, CacheManager::latest_branch_info_on(&conn))
1354        else {
1355            return Ok(None);
1356        };
1357        if !self.stores_intact(file_count, meta_generation) {
1358            return Ok(None);
1359        }
1360        let generation = generation + 1;
1361        laps.lap("snapshot");
1362
1363        // The resolver configs the last walk found, unchanged since.
1364        let Some(resolver_configs) = crate::dependency_resolve::ResolverConfigs::from_saved_list(
1365            root,
1366            &cache_dir,
1367            &self.config.vendored_patterns,
1368        ) else {
1369            return Ok(None);
1370        };
1371        if stored_digest.as_deref() != Some(resolver_configs.digest.as_str()) {
1372            return Ok(None);
1373        }
1374        laps.lap("configs");
1375
1376        // Git: still the branch last indexed; a HEAD that moved adds what it changed.
1377        let (branch, commit) = if in_git {
1378            let Ok((commit, branch)) = crate::git::head_commit_and_branch(root) else {
1379                return Ok(None);
1380            };
1381            (branch, Some(commit))
1382        } else {
1383            ("_default".to_string(), None)
1384        };
1385        // Only the branch whose rows the last run synced (its rows are then
1386        // updated one by one).
1387        if synced_branch.as_deref() != Some(branch.as_str()) || branch != indexed.branch {
1388            return Ok(None);
1389        }
1390        if let Some(commit) = &commit
1391            && *commit != indexed.commit_sha
1392        {
1393            let Ok(moved) = crate::git::diff_names(root, &indexed.commit_sha, commit) else {
1394                return Ok(None);
1395            };
1396            let mut extra: Vec<String> = Vec::new();
1397            for path in moved {
1398                match classify_named(root, &abs_root, Path::new(&path)) {
1399                    Named::Skip => {}
1400                    Named::Full => return Ok(None),
1401                    Named::Path(rel) => {
1402                        if named.binary_search(&rel).is_err() {
1403                            extra.push(rel);
1404                        }
1405                    }
1406                }
1407            }
1408            if !extra.is_empty() {
1409                status_jobs.push(spawn_status(extra.clone()));
1410                named.extend(extra);
1411                named.sort();
1412                named.dedup();
1413            }
1414        }
1415        if resolver_configs.lists_any_under(root, &named)
1416            || named.iter().any(|rel| {
1417                let dir = root.join(rel);
1418                dir.is_dir() && holds_resolver_config(&dir)
1419            })
1420        {
1421            return Ok(None);
1422        }
1423        let targets = Arc::new(Targets::new(&named));
1424        laps.lap("git");
1425
1426        // The rows at and under the named paths, and those paths as a full walk
1427        // would see them now.
1428        let stored = crate::meta_update::load_rows_under(&conn, &named)?;
1429        let Discovered {
1430            rels: found_rels,
1431            metas: found_metas,
1432            skipped_too_large,
1433            skipped_bytes_too_large,
1434            skipped_binary,
1435            ..
1436        } = self.discover_files_in(root, &stored, Some(Arc::clone(&targets)))?;
1437        let found: HashMap<&str, usize> = found_rels
1438            .iter()
1439            .enumerate()
1440            .map(|(i, rel)| (rel.as_str(), i))
1441            .collect();
1442        laps.lap("walk");
1443
1444        // What changed, by the rules of a full run.
1445        let num_threads = crate::models::resolve_thread_count(self.config.parallel_threads, 32);
1446        let pool = rayon::ThreadPoolBuilder::new()
1447            .num_threads(num_threads)
1448            .build()
1449            .context("Failed to create thread pool")?;
1450        let to_hash: Vec<usize> = (0..found_rels.len())
1451            .filter(|&i| {
1452                stored.get(&found_rels[i]).is_some_and(|row| {
1453                    !found_metas[i]
1454                        .as_ref()
1455                        .is_some_and(|md| row.stat_matches(md))
1456                })
1457            })
1458            .collect();
1459        let hashed: Vec<(usize, Option<String>)> = pool.install(|| {
1460            to_hash
1461                .par_iter()
1462                .map(|&i| {
1463                    let hash = std::fs::read(root.join(&found_rels[i]))
1464                        .ok()
1465                        .map(|b| self.hash_content(&b));
1466                    (i, hash)
1467                })
1468                .collect()
1469        });
1470        let mut found_status: Vec<FileStatus> = found_rels
1471            .iter()
1472            .map(|rel| {
1473                if stored.contains_key(rel) {
1474                    FileStatus::Unchanged
1475                } else {
1476                    FileStatus::Added
1477                }
1478            })
1479            .collect();
1480        for (i, hash) in hashed {
1481            found_status[i] = match hash {
1482                Some(h) if h == stored[&found_rels[i]].hash => FileStatus::Touched,
1483                _ => FileStatus::Modified,
1484            };
1485        }
1486        let rewritten =
1487            |i: usize| matches!(found_status[i], FileStatus::Added | FileStatus::Modified);
1488        // Indexed paths at or under a named path that the walk no longer yields.
1489        let deleted: Vec<&str> = {
1490            let mut d: Vec<&str> = stored
1491                .keys()
1492                .map(String::as_str)
1493                .filter(|p| !found.contains_key(p))
1494                .collect();
1495            d.sort_unstable();
1496            d
1497        };
1498        laps.lap("hash");
1499
1500        // Walk positions of the rewritten files.
1501        let to_place: Vec<(&str, Option<&crate::meta_update::StoredFile>)> = (0..found_rels.len())
1502            .filter(|&i| rewritten(i))
1503            .map(|i| (found_rels[i].as_str(), stored.get(&found_rels[i])))
1504            .collect();
1505        let deleted_ids: std::collections::HashSet<i64> =
1506            deleted.iter().map(|p| stored[*p].id).collect();
1507        let mut walk_order = crate::walk_order::WalkOrder::new(root);
1508        let Some(seq_of) = place_rewrites(&conn, &mut walk_order, &to_place, &deleted_ids)? else {
1509            return Ok(None);
1510        };
1511        drop(conn);
1512        let added = found_status
1513            .iter()
1514            .filter(|s| **s == FileStatus::Added)
1515            .count();
1516        let live_files = file_count - deleted.len() + added;
1517        laps.lap("order");
1518
1519        let mut rewrites = Vec::new();
1520        let mut rewrite_rows = HashMap::new();
1521        for (i, rel) in found_rels.iter().enumerate() {
1522            if rewritten(i) {
1523                rewrites.push(Rewrite {
1524                    rel: rel.clone(),
1525                    size: found_metas[i].as_ref().map_or(0, |md| md.size()),
1526                    walk_seq: seq_of[rel],
1527                });
1528                if let Some(row) = stored.get(rel) {
1529                    rewrite_rows.insert(rel.clone(), row.clone());
1530                }
1531            }
1532        }
1533        // Dirty flags of the named paths as `git status` lists them now (every other
1534        // row keeps the flag it has), joined only when meta.db is written. If git
1535        // fails, every named path counts as dirty: a superset is safe for freshness.
1536        let dirty_job = || -> DirtyFlags {
1537            let mut now_dirty: std::collections::HashSet<String> = std::collections::HashSet::new();
1538            let mut ok = true;
1539            for job in status_jobs {
1540                match job.join() {
1541                    Ok(Ok(paths)) => now_dirty.extend(paths),
1542                    _ => ok = false,
1543                }
1544            }
1545            let is_dirty = |rel: &str| !ok || now_dirty.contains(rel);
1546            let mut flags = DirtyFlags {
1547                rewrites: std::collections::HashSet::new(),
1548                touched: Vec::new(),
1549                flips: Vec::new(),
1550                git_dirty: in_git && (indexed.is_dirty || !ok || !now_dirty.is_empty()),
1551            };
1552            for (i, rel) in found_rels.iter().enumerate() {
1553                let dirty = is_dirty(rel);
1554                if rewritten(i) {
1555                    if dirty {
1556                        flags.rewrites.insert(rel.clone());
1557                    }
1558                } else if let Some(row) = stored.get(rel) {
1559                    if found_status[i] == FileStatus::Touched {
1560                        let (size, mtime) = found_metas[i]
1561                            .as_ref()
1562                            .map(|md| (md.size(), md.recorded_mtime_ns(run_start)))
1563                            .unwrap_or((0, 0));
1564                        flags.touched.push((row.id, size, mtime, dirty));
1565                    } else if dirty != row.dirty {
1566                        flags.flips.push((row.id, dirty));
1567                    }
1568                }
1569            }
1570            flags
1571        };
1572        // The branch's own hashes of the rewritten paths; every other file's row
1573        // matches its hash (the last run on this branch synced them all).
1574        let branch_hashes: HashMap<String, String> = {
1575            let conn = crate::cache::open_meta_db(cache_dir.join(crate::cache::META_DB))?;
1576            let mut stmt = conn.prepare(
1577                "SELECT fb.hash FROM file_branches fb
1578                 JOIN files f ON f.id = fb.file_id
1579                 JOIN branches b ON b.id = fb.branch_id
1580                 WHERE b.name = ? AND f.path = ?",
1581            )?;
1582            let mut out = HashMap::new();
1583            for r in &rewrites {
1584                let hash: Option<String> = rusqlite::OptionalExtension::optional(
1585                    stmt.query_row(rusqlite::params![branch, r.rel], |row| row.get(0)),
1586                )?;
1587                if let Some(hash) = hash {
1588                    out.insert(r.rel.clone(), hash);
1589                }
1590            }
1591            out
1592        };
1593        let skipped = (skipped_too_large, skipped_bytes_too_large, skipped_binary);
1594        let rewritten_count = rewrites.len();
1595        laps.lap("change_set");
1596
1597        let result = if deleted.is_empty() && rewrites.is_empty() {
1598            let flags = dirty_job();
1599            self.refresh_named(RefreshNamed {
1600                touched: &flags.touched,
1601                flips: &flags.flips,
1602                walk_moves: &[],
1603                branch: &branch,
1604                commit: commit.as_deref(),
1605                git_dirty: flags.git_dirty,
1606                live_files,
1607                skipped,
1608            })
1609            .map(Some)
1610        } else {
1611            let deleted_file_count = deleted.iter().filter(|p| !root.join(p).exists()).count();
1612            self.publish_delta(DeltaChanges {
1613                root,
1614                rewrites,
1615                deleted: deleted
1616                    .iter()
1617                    .map(|p| (p.to_string(), stored[*p].id))
1618                    .collect(),
1619                walk_moves: Vec::new(),
1620                dirty: Box::new(dirty_job),
1621                stored: rewrite_rows,
1622                sync_all_branch_rows: false,
1623                full_stats: false,
1624                live_files,
1625                breakdown_rest: (0, 0, live_files - rewritten_count),
1626                branch_hashes,
1627                branch: &branch,
1628                commit: commit.as_deref(),
1629                resolver_configs: &resolver_configs,
1630                run_start,
1631                generation,
1632                pool: &pool,
1633                deleted_file_count,
1634                skipped,
1635            })
1636        };
1637        laps.lap("publish");
1638        log::info!(
1639            "update_paths: {} named paths, {} rewritten, {} deleted in {} ms;{}",
1640            named.len(),
1641            rewritten_count,
1642            deleted.len(),
1643            started.elapsed().as_millis(),
1644            laps.line
1645        );
1646        result
1647    }
1648
1649    /// How long to wait for a running symbol pass to yield the database.
1650    ///
1651    /// One batch is 128 files, so a pass normally yields in well under a second.
1652    /// 10s covers a slow batch without making the caller wait out a whole pass.
1653    const SYMBOL_PASS_YIELD_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);
1654
1655    /// Wait for the detached background symbol pass to release `meta.db`.
1656    ///
1657    /// The pass (`rfx index-symbols-internal`) runs for minutes on a large repo and
1658    /// holds a SQLite write lock the whole time, but takes its own `indexing.lock`
1659    /// rather than the workspace `index.lock`. In 1.7.0 that meant every
1660    /// `index_project` and `rfx index` during the window failed with a raw
1661    /// `Failed to begin meta.db schema transaction: database is locked`.
1662    ///
1663    /// Asks the pass to stop at its next batch, then waits briefly. If it does not
1664    /// yield, returns [`ReflexError::SymbolIndexingInProgress`], which names the pid
1665    /// and the progress — never SQLite's error text.
1666    fn yield_to_symbol_pass(&self, cache_dir: &Path) -> Result<()> {
1667        use crate::background_indexer::BackgroundIndexer;
1668
1669        let Some(holder) = BackgroundIndexer::lock_holder(cache_dir) else {
1670            // Nothing running (or the lock was stale and has just been reaped).
1671            BackgroundIndexer::clear_cancel(cache_dir);
1672            return Ok(());
1673        };
1674
1675        log::info!(
1676            "Symbol indexing is running (pid {}); asking it to yield",
1677            holder.pid
1678        );
1679        let _ = BackgroundIndexer::request_cancel(cache_dir);
1680
1681        let deadline = std::time::Instant::now() + Self::SYMBOL_PASS_YIELD_TIMEOUT;
1682        // A pass that has nothing left to do exits within milliseconds: poll fast
1683        // first, then every 100 ms.
1684        let mut pause = std::time::Duration::from_millis(5);
1685        while std::time::Instant::now() < deadline {
1686            if !BackgroundIndexer::is_running(cache_dir) {
1687                BackgroundIndexer::clear_cancel(cache_dir);
1688                log::info!("Symbol indexing yielded; continuing");
1689                return Ok(());
1690            }
1691            std::thread::sleep(pause);
1692            pause = (pause * 2).min(std::time::Duration::from_millis(100));
1693        }
1694
1695        // It did not yield. Report what it is doing, then leave it alone.
1696        BackgroundIndexer::clear_cancel(cache_dir);
1697        let status = BackgroundIndexer::get_status(cache_dir).ok().flatten();
1698        Err(crate::errors::ReflexError::SymbolIndexingInProgress {
1699            pid: holder.pid,
1700            started_at: holder.started_clock(),
1701            processed: status.as_ref().map(|s| s.processed_files).unwrap_or(0),
1702            total: status.as_ref().map(|s| s.total_files).unwrap_or(0),
1703        }
1704        .into())
1705    }
1706
1707    /// Build or update the index with progress callback support
1708    pub fn index_with_callback(
1709        &self,
1710        root: impl AsRef<Path>,
1711        show_progress: bool,
1712        progress_callback: Option<ProgressCallback>,
1713    ) -> Result<IndexStats> {
1714        let root = root.as_ref();
1715        let stats = self.index_run(root, show_progress, progress_callback)?;
1716        // The rule files this run indexed under, once its index is published. A
1717        // crash before this write leaves the old record: the next check reports
1718        // the rules as changed and the next run writes it.
1719        let pending = self
1720            .rules_pending
1721            .lock()
1722            .unwrap_or_else(|e| e.into_inner())
1723            .take();
1724        if let Some(record) = pending {
1725            let conn = crate::cache::open_meta_db(self.cache.path().join(crate::cache::META_DB))?;
1726            crate::meta_update::set_statistic(
1727                &conn,
1728                RULE_FILES_KEY,
1729                &record,
1730                chrono::Utc::now().timestamp(),
1731            )?;
1732            crate::query::invalidate_caches(root);
1733        }
1734        Ok(stats)
1735    }
1736
1737    fn index_run(
1738        &self,
1739        root: &Path,
1740        show_progress: bool,
1741        progress_callback: Option<ProgressCallback>,
1742    ) -> Result<IndexStats> {
1743        log::info!("Indexing directory: {:?}", root);
1744        let mut laps = Laps::new();
1745
1746        // Exclusive workspace lock for the whole run. Two indexers streaming
1747        // into the same content.bin/trigrams.bin is how a reader ends up with a
1748        // short file. The OS drops the lock if this process dies.
1749        let cache_dir = self.cache.path().to_path_buf();
1750        let _index_lock = crate::atomic_write::IndexLock::acquire_with_timeout(
1751            &cache_dir,
1752            std::time::Duration::from_secs(self.config.lock_wait_secs),
1753        )?;
1754        // A previous indexer that died mid-write leaves `*.tmp` behind.
1755        crate::atomic_write::remove_stale_tmp(&cache_dir);
1756
1757        // Any open handle on the stores about to be rewritten is dropped now, and
1758        // again on every exit path below, so a query in this process never reads a
1759        // mix of old and new files and never keeps a memo of a stale verdict.
1760        crate::query::invalidate_caches(root);
1761        let _invalidate_on_exit = InvalidateOnDrop(root.to_path_buf());
1762
1763        // The detached symbol pass holds meta.db but NOT this lock, so acquiring
1764        // `index.lock` above proves nothing about SQLite. Yield to it here, before
1765        // `cache.init()` runs `BEGIN IMMEDIATE`, so a waiting agent sees progress
1766        // instead of `database is locked: Error code 5`.
1767        self.yield_to_symbol_pass(&cache_dir)?;
1768        laps.lap("lock");
1769
1770        // Configure thread pool for parallel processing.
1771        // 0 = auto (80% of available cores, up to 32 — the query pool's rule).
1772        // The pool reads, hashes, extracts imports and trigrams; the only serial
1773        // work left is streaming file bytes into content.bin. Peak memory is set
1774        // by the batch byte budget, not by the thread count.
1775        let num_threads = crate::models::resolve_thread_count(self.config.parallel_threads, 32);
1776
1777        log::info!(
1778            "Using {} threads for parallel indexing (out of {} available)",
1779            num_threads,
1780            std::thread::available_parallelism()
1781                .map(|n| n.get())
1782                .unwrap_or(4)
1783        );
1784
1785        // Refuse to write a cache another Reflex build owns.
1786        //
1787        // A `force` rebuild clears `.reflex/` first, so there is no meta.db left to
1788        // conflict with and this passes — taking ownership is exactly what force
1789        // means. Checked before `init()`, which is itself a write.
1790        self.cache.assert_writable(false)?;
1791
1792        // Whether the cache was last completed by a binary with this cache schema.
1793        // Read BEFORE `init()`: a cache from other code must be rebuilt in full, and
1794        // a new cache has nothing to reuse either way.
1795        let schema_ok = self.cache.check_schema_hash().unwrap_or(false);
1796        let extraction_ok = self.cache.check_extraction_hash().unwrap_or(false);
1797
1798        // Ensure cache is initialized. A cache this binary completed a run on has
1799        // every table: only `config.toml` may need recreating.
1800        if schema_ok && extraction_ok {
1801            self.cache.ensure_config()?;
1802        } else {
1803            self.cache.init()?;
1804        }
1805
1806        // Extraction code (parsers, resolution) changed since this cache was last
1807        // written: stored symbols are from other code, and every file's imports
1808        // are extracted again below.
1809        if !schema_ok || !extraction_ok {
1810            match self.cache.clear_symbol_cache() {
1811                Ok(n) if n > 0 => {
1812                    log::info!("Cleared {} symbol cache rows written by other code", n)
1813                }
1814                Ok(_) => {}
1815                Err(e) => log::warn!("Failed to clear the symbol cache: {}", e),
1816            }
1817        }
1818
1819        // Files modified at or after this run began record an unknown mtime (0), so
1820        // a write racing the read is caught by hash on the next check. The threshold
1821        // is the mtime of a file written now, on the same (coarse) clock that stamps
1822        // the files: a write in the same tick as a read could otherwise carry an
1823        // mtime just below a process-clock `run_start` and be trusted forever.
1824        let run_start = self.run_marker(&cache_dir);
1825
1826        laps.lap("init");
1827
1828        let pool = rayon::ThreadPoolBuilder::new()
1829            .num_threads(num_threads)
1830            .build()
1831            .context("Failed to create thread pool")?;
1832
1833        // Step 1: walk the tree. In parallel, load what meta.db holds (one row per
1834        // path of the last indexed tree), ask git for the branch and the dirty
1835        // paths, and find the resolver configs (each is its own walk or subprocess).
1836        let meta_path = cache_dir.join(crate::cache::META_DB);
1837        let phase_start = Instant::now();
1838        let (git_state, resolver_configs, meta_read, walked) = std::thread::scope(|scope| {
1839            let git = scope.spawn(|| {
1840                let t = Instant::now();
1841                let r = crate::git::get_git_state_optional(root);
1842                (r, t.elapsed().as_millis())
1843            });
1844            let configs = scope.spawn(|| {
1845                let t = Instant::now();
1846                let r = crate::dependency_resolve::ResolverConfigs::discover(
1847                    root,
1848                    &self.config.vendored_patterns,
1849                );
1850                (r, t.elapsed().as_millis())
1851            });
1852            // Available disk space (a `df` subprocess), now that the cache exists.
1853            let disk = scope.spawn(|| self.check_disk_space(root));
1854            let meta_read = scope.spawn(|| -> Result<_> {
1855                let t = Instant::now();
1856                let conn = crate::cache::open_meta_db(&meta_path)?;
1857                let rows = crate::meta_update::load_stored_files(&conn)?;
1858                log::debug!("meta.db rows read in {} ms", t.elapsed().as_millis());
1859                Ok((
1860                    rows,
1861                    crate::meta_update::get_statistic(&conn, RESOLVER_DIGEST_KEY)?,
1862                    crate::meta_update::get_statistic(&conn, INDEX_GENERATION_KEY)?
1863                        .and_then(|g| g.parse::<u64>().ok()),
1864                    crate::meta_update::get_statistic(
1865                        &conn,
1866                        crate::meta_update::SYNCED_BRANCH_KEY,
1867                    )?,
1868                    crate::meta_update::get_statistic(&conn, RULE_FILES_KEY)?,
1869                ))
1870            });
1871            let t = Instant::now();
1872            let walked = self.walk_candidates(root, None);
1873            let walk_ms = t.elapsed().as_millis();
1874            let (git, configs) = (git.join(), configs.join());
1875            log::info!(
1876                "discovery phase: walk {} ms, git {} ms, resolver configs {} ms",
1877                walk_ms,
1878                git.as_ref().map_or(0, |g| g.1),
1879                configs.as_ref().map_or(0, |c| c.1)
1880            );
1881            let disk = disk
1882                .join()
1883                .unwrap_or_else(|_| Err(anyhow::anyhow!("disk space check panicked")));
1884            (
1885                git.map(|g| g.0),
1886                configs.map(|c| c.0),
1887                meta_read.join(),
1888                walked.and_then(|w| disk.map(|()| w)),
1889            )
1890        });
1891        let (stored, stored_digest, meta_generation, synced_branch, stored_rules) =
1892            meta_read.map_err(|_| anyhow::anyhow!("meta.db read thread panicked"))??;
1893        // A non-code file is text only when it holds no NUL byte; one whose stat
1894        // matches its row was checked when it was indexed.
1895        let discovered = walked.map(|w| Self::drop_binaries(root, w, &stored, Some(&pool)));
1896        // The snapshot currently published, and the generation the next publish takes.
1897        let prev_manifest = crate::snapshot::read_manifest(&cache_dir).ok().flatten();
1898        let generation = prev_manifest
1899            .as_ref()
1900            .map(|m| m.generation)
1901            .max(meta_generation)
1902            .unwrap_or(0)
1903            + 1;
1904        let git_state = git_state.map_err(|_| anyhow::anyhow!("git state thread panicked"))??;
1905        let resolver_configs =
1906            resolver_configs.map_err(|_| anyhow::anyhow!("resolver config walk panicked"))?;
1907        // For `update_paths`, which parses the same configs without walking.
1908        if let Err(e) = resolver_configs.save_list(root, &cache_dir) {
1909            log::warn!("Failed to save the resolver config list: {:#}", e);
1910        }
1911        let Discovered {
1912            rels,
1913            sizes,
1914            metas,
1915            skipped_too_large,
1916            skipped_bytes_too_large,
1917            skipped_binary,
1918        } = discovered?;
1919        let total_files = rels.len();
1920        laps.lap("discover");
1921        log::info!(
1922            "Discovered {} files to index ({} skipped: too large, {} binary) in {} ms",
1923            total_files,
1924            skipped_too_large,
1925            skipped_binary,
1926            phase_start.elapsed().as_millis()
1927        );
1928
1929        let branch = git_state
1930            .as_ref()
1931            .map(|s| s.branch.clone())
1932            .unwrap_or_else(|| "_default".to_string());
1933        if let Some(ref state) = git_state {
1934            log::info!(
1935                "Git state: branch='{}', commit='{}', dirty={}",
1936                state.branch,
1937                state.commit,
1938                state.dirty
1939            );
1940        } else {
1941            log::info!("Not a git repository, using default branch");
1942        }
1943        let commit = git_state.as_ref().map(|s| s.commit.clone());
1944        let dirty_paths: std::collections::HashSet<String> = git_state
1945            .as_ref()
1946            .map(|s| s.dirty_paths.clone())
1947            .unwrap_or_default();
1948        let rules = rule_files_record(root, &dirty_paths);
1949        *self.rules_pending.lock().unwrap_or_else(|e| e.into_inner()) =
1950            (stored_rules.as_deref() != Some(rules.as_str())).then_some(rules);
1951
1952        // The branch's own hashes: the basis of the "new / modified / unchanged"
1953        // breakdown, as before stable ids.
1954        let in_sync = synced_branch.as_deref() == Some(branch.as_str());
1955        let existing_hashes = if in_sync {
1956            BranchHashes::Synced(&stored)
1957        } else {
1958            BranchHashes::Loaded(self.cache.load_hashes_for_branch(&branch)?)
1959        };
1960        laps.lap("branch_hashes");
1961
1962        // Step 2: what changed. A file whose (size, mtime) match its row is
1963        // unchanged without being read; the others are read and hashed.
1964        let classify_start = Instant::now();
1965        // Each walked path's row, looked up once.
1966        let rows_of: Vec<Option<&crate::meta_update::StoredFile>> =
1967            rels.iter().map(|rel| stored.get(rel)).collect();
1968        let to_hash: Vec<usize> = (0..total_files)
1969            .filter(|&i| {
1970                rows_of[i]
1971                    .is_some_and(|row| !metas[i].as_ref().is_some_and(|md| row.stat_matches(md)))
1972            })
1973            .collect();
1974        let hashed: Vec<(usize, Option<String>)> = pool.install(|| {
1975            to_hash
1976                .par_iter()
1977                .map(|&i| {
1978                    (
1979                        i,
1980                        std::fs::read(root.join(&rels[i]))
1981                            .ok()
1982                            .map(|b| self.hash_content(&b)),
1983                    )
1984                })
1985                .collect()
1986        });
1987        let mut status: Vec<FileStatus> = rows_of
1988            .iter()
1989            .map(|row| {
1990                if row.is_some() {
1991                    FileStatus::Unchanged
1992                } else {
1993                    FileStatus::Added
1994                }
1995            })
1996            .collect();
1997        for (i, hash) in hashed {
1998            status[i] = match (hash, rows_of[i]) {
1999                (Some(h), Some(row)) if h == row.hash => FileStatus::Touched,
2000                _ => FileStatus::Modified,
2001            };
2002        }
2003        // Every stored path was walked (the usual case) when as many walked paths
2004        // have a row as there are rows: nothing is gone.
2005        let walked_with_row = status.iter().filter(|s| **s != FileStatus::Added).count();
2006        let gone: Vec<&String> = if walked_with_row == stored.len() {
2007            Vec::new()
2008        } else {
2009            let discovered_set: std::collections::HashSet<&str> =
2010                rels.iter().map(String::as_str).collect();
2011            stored
2012                .keys()
2013                .filter(|p| !discovered_set.contains(p.as_str()))
2014                .collect()
2015        };
2016        // Counted as "deleted" only when the file is gone from disk (a file that is
2017        // merely no longer indexable was never counted), as before.
2018        let deleted_file_count = gone.iter().filter(|p| !root.join(p).exists()).count();
2019        let added = status.iter().filter(|s| **s == FileStatus::Added).count();
2020        let modified = status
2021            .iter()
2022            .filter(|s| **s == FileStatus::Modified)
2023            .count();
2024        log::info!(
2025            "phase classify: {} added, {} modified, {} touched, {} gone; {} files hashed in {} ms",
2026            added,
2027            modified,
2028            status.iter().filter(|s| **s == FileStatus::Touched).count(),
2029            gone.len(),
2030            to_hash.len(),
2031            classify_start.elapsed().as_millis()
2032        );
2033
2034        // Every dependency row is rewritten when the extraction code, the schema or
2035        // any resolver config (go.mod, tsconfig.json, ...) changed.
2036        let full_deps = !schema_ok
2037            || !extraction_ok
2038            || stored_digest.as_deref() != Some(resolver_configs.digest.as_str());
2039        let content_changed = added > 0 || modified > 0 || !gone.is_empty();
2040        laps.lap("classify");
2041        let stores_ok = schema_ok && self.stores_intact(stored.len(), meta_generation);
2042        laps.lap("snapshot");
2043
2044        if !content_changed && !full_deps && stores_ok {
2045            log::info!("No files changed - skipping index rebuild");
2046            log::info!("index run phases:{}", laps.line);
2047            return self.refresh_unchanged(RefreshUnchanged {
2048                rels: &rels,
2049                metas: &metas,
2050                status: &status,
2051                rows: &rows_of,
2052                existing_hashes: &existing_hashes,
2053                dirty_paths: &dirty_paths,
2054                branch: &branch,
2055                in_sync,
2056                commit: commit.as_deref(),
2057                git_dirty: git_state.as_ref().map(|s| s.dirty),
2058                run_start,
2059                skipped: (skipped_too_large, skipped_bytes_too_large, skipped_binary),
2060            });
2061        }
2062        if content_changed && !full_deps && stores_ok {
2063            // The change set from the whole walk: walk positions planned over every
2064            // file, dirty flags from `git status`.
2065            let seqs = crate::meta_update::plan_walk_seq(
2066                &rels
2067                    .iter()
2068                    .map(|rel| stored.get(rel).map(|s| s.walk_seq))
2069                    .collect::<Vec<_>>(),
2070            );
2071            let mut rewrites = Vec::new();
2072            let mut dirty_rewrites = std::collections::HashSet::new();
2073            let mut rewrite_rows = HashMap::new();
2074            let mut branch_hashes = HashMap::new();
2075            let (mut touched, mut walk_moves, mut flips) = (Vec::new(), Vec::new(), Vec::new());
2076            let mut rest: Vec<(&str, &str)> = Vec::new();
2077            for i in 0..total_files {
2078                let rel = &rels[i];
2079                let dirty = dirty_paths.contains(rel);
2080                match (status[i], stored.get(rel)) {
2081                    (FileStatus::Added | FileStatus::Modified, row) => {
2082                        rewrites.push(Rewrite {
2083                            rel: rel.clone(),
2084                            size: sizes[i],
2085                            walk_seq: seqs[i],
2086                        });
2087                        if dirty {
2088                            dirty_rewrites.insert(rel.clone());
2089                        }
2090                        if let Some(row) = row {
2091                            rewrite_rows.insert(rel.clone(), row.clone());
2092                        }
2093                        if let Some(h) = existing_hashes.get(rel) {
2094                            branch_hashes.insert(rel.clone(), h.to_string());
2095                        }
2096                    }
2097                    (_, Some(row)) => {
2098                        if seqs[i] != row.walk_seq {
2099                            walk_moves.push((row.id, seqs[i]));
2100                        }
2101                        if status[i] == FileStatus::Touched {
2102                            let (size, mtime) = metas[i]
2103                                .as_ref()
2104                                .map(|md| (md.size(), md.recorded_mtime_ns(run_start)))
2105                                .unwrap_or((0, 0));
2106                            touched.push((row.id, size, mtime, dirty));
2107                        } else if dirty != row.dirty {
2108                            flips.push((row.id, dirty));
2109                        }
2110                        rest.push((rel.as_str(), row.hash.as_str()));
2111                    }
2112                    (_, None) => unreachable!("an unchanged file has a row"),
2113                }
2114            }
2115            let git_dirty = git_state.as_ref().map(|s| s.dirty).unwrap_or(false);
2116            let changes = DeltaChanges {
2117                root,
2118                rewrites,
2119                deleted: gone.iter().map(|p| ((*p).clone(), stored[*p].id)).collect(),
2120                walk_moves,
2121                dirty: Box::new(move || DirtyFlags {
2122                    rewrites: dirty_rewrites,
2123                    touched,
2124                    flips,
2125                    git_dirty,
2126                }),
2127                stored: rewrite_rows,
2128                sync_all_branch_rows: !in_sync,
2129                full_stats: true,
2130                live_files: total_files,
2131                breakdown_rest: breakdown(rest.into_iter(), &existing_hashes),
2132                branch_hashes,
2133                branch: &branch,
2134                commit: commit.as_deref(),
2135                resolver_configs: &resolver_configs,
2136                run_start,
2137                generation,
2138                pool: &pool,
2139                deleted_file_count,
2140                skipped: (skipped_too_large, skipped_bytes_too_large, skipped_binary),
2141            };
2142            match self.publish_delta(changes)? {
2143                Some(stats) => return Ok(stats),
2144                None => log::info!("The delta would pass its limits - building a new base"),
2145            }
2146        }
2147        if full_deps {
2148            log::info!(
2149                "Resolver configs, extraction code or schema changed - re-extracting every file's imports"
2150            );
2151        }
2152
2153        // Step 3: Build trigram index + content store from every file (the stores
2154        // are rewritten in full). Imports are extracted only where they may have
2155        // changed.
2156        let mut files_indexed = 0;
2157        let mut indexed: Vec<IndexedFile> = Vec::with_capacity(total_files);
2158
2159        // Initialize trigram builder and content store. The builder spills each
2160        // batch to `<cache>/trigram_temp/` only when there is more than one batch;
2161        // a single batch stays in memory and the directory is never created.
2162        let mut trigram_builder = TrigramIndexBuilder::new(self.cache.path().join("trigram_temp"));
2163        let mut content_writer = ContentWriter::new();
2164
2165        // Initialize content writer to start streaming writes immediately
2166        let (content_name, trigrams_name, plan_name) = crate::snapshot::base_file_names(generation);
2167        let content_path = cache_dir.join(&content_name);
2168        content_writer
2169            .init(content_path.clone())
2170            .context("Failed to initialize content writer")?;
2171
2172        // Create progress bar (only if requested via --progress flag)
2173        let pb = if show_progress {
2174            let pb = ProgressBar::new(total_files as u64);
2175            pb.set_draw_target(ProgressDrawTarget::stderr());
2176            pb.set_style(
2177                ProgressStyle::default_bar()
2178                    .template("[{elapsed_precise}] [{bar:40.cyan/blue}] {pos}/{len} files ({percent}%) {msg}")
2179                    .unwrap()
2180                    .progress_chars("=>-")
2181            );
2182            // Force updates every 100ms to ensure progress is visible
2183            pb.enable_steady_tick(std::time::Duration::from_millis(100));
2184            pb
2185        } else {
2186            ProgressBar::hidden()
2187        };
2188
2189        // Atomic counter for thread-safe progress updates
2190        let progress_counter = Arc::new(AtomicU64::new(0));
2191        // Shared status message for progress callback
2192        let progress_status = Arc::new(Mutex::new("Indexing files...".to_string()));
2193
2194        let batch_phase_start = Instant::now();
2195        let mut pool_ms = 0u128;
2196        let mut flush_ms = 0u128;
2197
2198        // A merge: when the published stores match meta.db, a file whose bytes
2199        // are unchanged takes its text from them instead of the disk. Same text,
2200        // same order, same builder: the new base is byte-identical to a build
2201        // that reads every file.
2202        let merge_source: Option<(crate::snapshot::IndexSnapshot, HashMap<String, u32>)> =
2203            if stores_ok {
2204                crate::snapshot::IndexSnapshot::open(&cache_dir)
2205                    .ok()
2206                    .map(|snapshot| {
2207                        let ids = snapshot
2208                            .live_ids()
2209                            .filter_map(|id| {
2210                                Some((snapshot.get_file_path(id)?.to_str()?.to_string(), id))
2211                            })
2212                            .collect();
2213                        (snapshot, ids)
2214                    })
2215            } else {
2216                None
2217            };
2218        let from_snapshot = std::sync::atomic::AtomicUsize::new(0);
2219
2220        // Spawn a background thread to update progress bar and call callback during parallel processing
2221        let counter_for_thread = Arc::clone(&progress_counter);
2222        let status_for_thread = Arc::clone(&progress_status);
2223        let pb_clone = pb.clone();
2224        let callback_for_thread = progress_callback.clone();
2225        let total_files_for_thread = total_files;
2226        let progress_thread = if show_progress || callback_for_thread.is_some() {
2227            Some(std::thread::spawn(move || {
2228                loop {
2229                    let count = counter_for_thread.load(Ordering::Relaxed);
2230                    pb_clone.set_position(count);
2231
2232                    // Call progress callback if provided
2233                    if let Some(ref callback) = callback_for_thread {
2234                        let status = status_for_thread.lock().unwrap().clone();
2235                        callback(count as usize, total_files_for_thread, status);
2236                    }
2237
2238                    if count >= total_files_for_thread as u64 {
2239                        break;
2240                    }
2241                    std::thread::sleep(std::time::Duration::from_millis(50));
2242                }
2243            }))
2244        } else {
2245            None
2246        };
2247
2248        // Process files in batches to bound memory: a batch holds at most
2249        // `max_files` files and about `max_bytes` bytes of text, so the pool's
2250        // in-flight contents and the trigram build's postings stay bounded on
2251        // any tree.
2252        let (max_files, max_bytes) = self.batch_limits();
2253        let batches = plan_batches(&sizes, max_files, max_bytes);
2254        let num_batches = batches.len();
2255        let spill_to_disk = num_batches > 1;
2256        log::info!(
2257            "Processing {} files in {} batches (<= {} files, ~{} MB each)",
2258            total_files,
2259            num_batches,
2260            max_files,
2261            max_bytes >> 20
2262        );
2263
2264        for (batch_idx, batch_range) in batches.into_iter().enumerate() {
2265            let batch_start = batch_range.start;
2266            let batch_len = batch_range.len();
2267            log::info!(
2268                "Processing batch {}/{} ({} files)",
2269                batch_idx + 1,
2270                num_batches,
2271                batch_len
2272            );
2273            let pool_start = Instant::now();
2274
2275            // Process files in parallel using rayon with custom thread pool.
2276            // `map_init` gives each worker one reusable trigram sort buffer.
2277            let counter_clone = Arc::clone(&progress_counter);
2278            let tsconfigs = &resolver_configs.tsconfigs;
2279            let ctx = ProcessCtx {
2280                root,
2281                rels: &rels,
2282                stored: &stored,
2283                run_start,
2284                full_deps,
2285                tsconfigs,
2286            };
2287            let results: Vec<Option<FileProcessingResult>> = pool.install(|| {
2288                batch_range
2289                    .into_par_iter()
2290                    .map_init(Vec::<u64>::new, |trigram_scratch, i| {
2291                        let reused = match &merge_source {
2292                            Some((snapshot, ids))
2293                                if matches!(
2294                                    status[i],
2295                                    FileStatus::Unchanged | FileStatus::Touched
2296                                ) =>
2297                            {
2298                                ids.get(&rels[i]).and_then(|&id| {
2299                                    self.reuse_file(
2300                                        &ctx,
2301                                        snapshot,
2302                                        id,
2303                                        i,
2304                                        &metas[i],
2305                                        trigram_scratch,
2306                                    )
2307                                })
2308                            }
2309                            _ => None,
2310                        };
2311                        let result = match reused {
2312                            Some(r) => {
2313                                from_snapshot.fetch_add(1, Ordering::Relaxed);
2314                                Some(r)
2315                            }
2316                            None => self.process_file(&ctx, i, trigram_scratch),
2317                        };
2318                        counter_clone.fetch_add(1, Ordering::Relaxed);
2319                        result
2320                    })
2321                    .collect()
2322            });
2323            pool_ms += pool_start.elapsed().as_millis();
2324            // A merge copied this batch's text out of the old stores: let their pages
2325            // go, or the whole old base ends up resident by the last batch.
2326            if let Some((snapshot, _)) = &merge_source {
2327                snapshot.release_pages();
2328            }
2329
2330            // Process batch results immediately (streaming approach to minimize memory)
2331            for (offset, result) in results.into_iter().enumerate() {
2332                let Some(result) = result else { continue };
2333                let i = batch_start + offset;
2334                // The normalized (forward-slash, relative) path everywhere, so the
2335                // trigram index and content store agree with the database and the
2336                // downstream filters, regardless of host separator.
2337                let normalized_pathbuf = PathBuf::from(&rels[i]);
2338
2339                // Register the file with the trigram builder (assigns file_id in
2340                // discovery order) and hand it the postings extracted in the pool.
2341                let _file_id =
2342                    trigram_builder.add_file(normalized_pathbuf.clone(), result.trigram_run);
2343
2344                // Add to content store
2345                content_writer.add_file(normalized_pathbuf, &result.content);
2346
2347                files_indexed += 1;
2348                indexed.push(IndexedFile {
2349                    index: i,
2350                    hash: result.hash,
2351                    language: result.language,
2352                    line_count: result.line_count,
2353                    size: result.size,
2354                    mtime_ns: result.mtime_ns,
2355                    imports: result.imports,
2356                });
2357            }
2358
2359            // Build this batch's posting lists (sharded, parallel) into a partial.
2360            let flush_msg = format!(
2361                "Building trigram batch {}/{}...",
2362                batch_idx + 1,
2363                num_batches
2364            );
2365            if show_progress {
2366                pb.set_message(flush_msg.clone());
2367            }
2368            *progress_status.lock().unwrap() = flush_msg;
2369            let flush_start = Instant::now();
2370            trigram_builder
2371                .flush_batch(&pool, spill_to_disk)
2372                .context("Failed to build trigram batch")?;
2373            flush_ms += flush_start.elapsed().as_millis();
2374        }
2375        log::info!(
2376            "phase read+extract: {} ms in pool, {} ms building trigram batches, {} ms total; {} files from the published stores",
2377            pool_ms,
2378            flush_ms,
2379            batch_phase_start.elapsed().as_millis(),
2380            from_snapshot.load(Ordering::Relaxed)
2381        );
2382        drop(merge_source);
2383
2384        // Wait for progress thread to finish
2385        if let Some(thread) = progress_thread {
2386            let _ = thread.join();
2387        }
2388
2389        // Update progress bar to final count
2390        if show_progress {
2391            let final_count = progress_counter.load(Ordering::Relaxed);
2392            pb.set_position(final_count);
2393        }
2394
2395        // Update progress bar message for post-processing
2396        *progress_status.lock().unwrap() = "Writing file metadata to database...".to_string();
2397        if show_progress {
2398            pb.set_message("Writing file metadata to database...".to_string());
2399        }
2400
2401        // Breakdown against the branch's own hashes (unchanged since stable ids).
2402        let (new_file_count, modified_file_count, unchanged_file_count) = breakdown(
2403            indexed
2404                .iter()
2405                .map(|f| (rels[f.index].as_str(), f.hash.as_str())),
2406            &existing_hashes,
2407        );
2408
2409        log::info!("Indexed {} files", files_indexed);
2410
2411        // Step 4: Write trigram index.
2412        // Crash-safe write: the builder streams into `trigrams.<g>.bin.tmp`, syncs,
2413        // then renames it into place (see `atomic_write`). Nothing names the new
2414        // files until the manifest below does, so a crash leaves the previous
2415        // snapshot as it was.
2416        *progress_status.lock().unwrap() = "Writing trigram index...".to_string();
2417        if show_progress {
2418            pb.set_message("Writing trigram index...".to_string());
2419        }
2420        let trigrams_path = cache_dir.join(&trigrams_name);
2421        let write_start = Instant::now();
2422        trigram_builder
2423            .write_with_plan(&pool, &trigrams_path, Some(&cache_dir.join(&plan_name)))
2424            .context("Failed to write trigram index")?;
2425        log::info!(
2426            "phase trigram write: {} trigrams, {} files, {} ms",
2427            trigram_builder.trigram_count(),
2428            trigram_builder.file_count(),
2429            write_start.elapsed().as_millis()
2430        );
2431
2432        // Step 5: Finalize content store (already been writing incrementally)
2433        *progress_status.lock().unwrap() = "Finalizing content store...".to_string();
2434        if show_progress {
2435            pb.set_message("Finalizing content store...".to_string());
2436        }
2437        content_writer
2438            .finalize_if_needed()
2439            .context("Failed to finalize content store")?;
2440        log::info!(
2441            "Wrote {} files ({} bytes) to {}",
2442            content_writer.file_count(),
2443            content_writer.content_size(),
2444            content_name
2445        );
2446
2447        // Step 6: Publish. The manifest rename is the commit point of the stores;
2448        // meta.db follows in one transaction that records the same generation.
2449        // A crash between the two leaves a manifest ahead of meta.db: the changed
2450        // files read as stale (their rows are older than the stores), and the next
2451        // run sees the generations differ and rebuilds.
2452        let manifest = crate::snapshot::Manifest::for_base(
2453            generation,
2454            crate::snapshot::SegmentFiles {
2455                content: content_name.clone(),
2456                trigrams: trigrams_name.clone(),
2457                plan: Some(plan_name.clone()),
2458                files: content_writer.file_count() as u64,
2459                content_bytes: std::fs::metadata(&content_path)?.len(),
2460                trigrams_bytes: std::fs::metadata(&trigrams_path)?.len(),
2461            },
2462            trigram_builder.trigram_count() as u64,
2463            content_writer.content_size() as u64,
2464        );
2465        self.abort_point("base-files");
2466        crate::snapshot::write_manifest(&cache_dir, &manifest)?;
2467        crate::snapshot::link_fixed_names(&cache_dir, &manifest);
2468        self.abort_point("base-manifest");
2469
2470        let meta_start = Instant::now();
2471        let present: Vec<usize> = indexed.iter().map(|f| f.index).collect();
2472        self.write_meta(MetaWrite {
2473            root,
2474            rels: &rels,
2475            present: &present,
2476            written: &indexed,
2477            metas: &metas,
2478            run_start,
2479            stored: &stored,
2480            status: &status,
2481            dirty_paths: &dirty_paths,
2482            branch: &branch,
2483            commit: commit.as_deref(),
2484            resolver_configs: &resolver_configs,
2485            full_deps,
2486            generation,
2487        })?;
2488        self.abort_point("base-meta");
2489
2490        // Update branch metadata
2491        self.cache.update_branch_metadata(
2492            &branch,
2493            commit.as_deref(),
2494            indexed.len(),
2495            git_state.as_ref().map(|s| s.dirty).unwrap_or(false),
2496        )?;
2497
2498        // Force WAL checkpoint to ensure background processes see all committed data
2499        // This is critical when spawning background symbol indexer immediately after
2500        self.cache
2501            .checkpoint_wal()
2502            .context("Failed to checkpoint WAL")?;
2503        log::info!(
2504            "phase meta.db (files, branches, dependencies, exports): {} ms",
2505            meta_start.elapsed().as_millis()
2506        );
2507
2508        // Readers in this process reopen; files no manifest names any more go.
2509        crate::query::invalidate_caches(root);
2510        crate::snapshot::remove_unreferenced(&cache_dir, &manifest, prev_manifest.as_ref());
2511
2512        // Step 7: Update SQLite statistics from database totals (branch-aware)
2513        *progress_status.lock().unwrap() = "Updating statistics...".to_string();
2514        if show_progress {
2515            pb.set_message("Updating statistics...".to_string());
2516        }
2517        // Update stats for current branch only
2518        self.cache.update_stats(&branch)?;
2519
2520        // Update schema hash to mark cache as compatible with current binary
2521        self.cache.update_schema_hash()?;
2522        self.cache.update_extraction_hash()?;
2523
2524        pb.finish_with_message("Indexing complete");
2525
2526        // Return stats with incremental breakdown (the branch's rows were synced
2527        // above: count from `files` alone).
2528        let mut stats = self.cache.stats_synced()?;
2529        stats.new_files = new_file_count;
2530        stats.modified_files = modified_file_count;
2531        stats.deleted_files = deleted_file_count;
2532        stats.unchanged_files = unchanged_file_count;
2533        stats.skipped_too_large = skipped_too_large;
2534        stats.skipped_bytes_too_large = skipped_bytes_too_large;
2535        stats.skipped_binary = skipped_binary;
2536        log::info!(
2537            "Indexing complete: {} files (new={}, modified={}, unchanged={})",
2538            stats.total_files,
2539            new_file_count,
2540            modified_file_count,
2541            unchanged_file_count
2542        );
2543
2544        Ok(stats)
2545    }
2546
2547    /// Apply a change set as a delta on the published base: the rewritten files
2548    /// are read, the delta is rebuilt from them and the previous delta's files the
2549    /// change set leaves alone, and the base files they supersede (or that are
2550    /// gone) are tombstoned. `meta.db` changes only the rows the change set names,
2551    /// in one transaction. `Ok(None)` when the delta would pass its limits (nothing
2552    /// is written): the caller then builds a new base (a merge).
2553    fn publish_delta(&self, c: DeltaChanges<'_>) -> Result<Option<IndexStats>> {
2554        use crate::snapshot::{IndexSnapshot, Manifest, SegmentFiles};
2555        use rusqlite::OptionalExtension;
2556        let cache_dir = self.cache.path().to_path_buf();
2557        let snapshot = IndexSnapshot::open(&cache_dir)?;
2558        let Some(prev) = snapshot.manifest().cloned() else {
2559            return Ok(None);
2560        };
2561        let base_len = snapshot.base_len();
2562        let delta_start = Instant::now();
2563        let mut laps = Laps::new();
2564
2565        let rels: Vec<String> = c.rewrites.iter().map(|r| r.rel.clone()).collect();
2566        let delta_len = snapshot.delta_len();
2567        // Ids below this are base and delta files, tombstoned in place; the recent
2568        // segment above them is rebuilt.
2569        let stable_end = base_len + delta_len;
2570
2571        // Store entries that go: every rewritten and every deleted path.
2572        let gone: std::collections::HashSet<String> = rels
2573            .iter()
2574            .cloned()
2575            .chain(c.deleted.iter().map(|(p, _)| p.clone()))
2576            .collect();
2577        let path_of = |id: u32| -> Option<&str> { snapshot.get_file_path(id)?.to_str() };
2578        let (mut new_dead_base, mut new_dead_delta) = (Vec::new(), Vec::new());
2579        for id in 0..stable_end {
2580            if path_of(id).is_some_and(|p| gone.contains(p)) {
2581                if id < base_len {
2582                    new_dead_base.push(id);
2583                } else {
2584                    new_dead_delta.push(id);
2585                }
2586            }
2587        }
2588        let kept_in = |range: std::ops::Range<u32>| -> Vec<(String, u32)> {
2589            range
2590                .filter_map(|id| {
2591                    let p = path_of(id)?;
2592                    (!gone.contains(p)).then(|| (p.to_string(), id))
2593                })
2594                .collect()
2595        };
2596        let delta_kept = kept_in(base_len..stable_end);
2597        let recent_kept = kept_in(stable_end..snapshot.id_bound());
2598
2599        // The limits, from stat sizes and entry-table lengths, before anything is
2600        // read (a merge reads the files itself).
2601        let len_of = |kept: &[(String, u32)]| -> u64 {
2602            kept.iter()
2603                .map(|(_, id)| snapshot.file_len(*id).unwrap_or(0))
2604                .sum()
2605        };
2606        let rewrite_bytes: u64 = c.rewrites.iter().map(|r| r.size).sum();
2607        let (delta_kept_bytes, recent_kept_bytes) = (len_of(&delta_kept), len_of(&recent_kept));
2608        let (max_files, max_bytes) = self.merge_limits(prev.live_corpus_bytes);
2609        let all_files = c.rewrites.len() + delta_kept.len() + recent_kept.len();
2610        let all_bytes = rewrite_bytes + delta_kept_bytes + recent_kept_bytes;
2611        if all_files > max_files || all_bytes > max_bytes {
2612            log::info!(
2613                "Delta of {} files / ~{} bytes passes its limits ({} files / {} bytes)",
2614                all_files,
2615                all_bytes,
2616                max_files,
2617                max_bytes
2618            );
2619            return Ok(None);
2620        }
2621        // The recent segment is rebuilt by every update; past its own limits it is
2622        // folded, with the delta's live files, into a new delta.
2623        let (recent_max_files, recent_max_bytes) = self.recent_limits(max_files, max_bytes);
2624        let fold = c.rewrites.len() + recent_kept.len() > recent_max_files
2625            || rewrite_bytes + recent_kept_bytes > recent_max_bytes;
2626
2627        // Read the rewritten files.
2628        let ctx = ProcessCtx {
2629            root: c.root,
2630            rels: &rels,
2631            stored: &c.stored,
2632            run_start: c.run_start,
2633            full_deps: false,
2634            tsconfigs: &c.resolver_configs.tsconfigs,
2635        };
2636        let mut read: Vec<Option<FileProcessingResult>> = c.pool.install(|| {
2637            (0..rels.len())
2638                .into_par_iter()
2639                .map_init(Vec::<u64>::new, |scratch, i| {
2640                    self.process_file(&ctx, i, scratch)
2641                })
2642                .collect()
2643        });
2644        laps.lap("read");
2645
2646        // An unreadable file is gone, as in a full run: its row goes too.
2647        let mut deleted = c.deleted;
2648        let mut live_files = c.live_files;
2649        for (k, r) in read.iter().enumerate() {
2650            if r.is_none() {
2651                live_files -= 1;
2652                if let Some(row) = c.stored.get(&rels[k]) {
2653                    deleted.push((rels[k].clone(), row.id));
2654                }
2655            }
2656        }
2657
2658        // The new segment: the read files and the kept ones, in walk order.
2659        enum Source {
2660            Fresh(usize),
2661            Old(u32),
2662        }
2663        let kept: Vec<(String, u32)> = if fold {
2664            delta_kept.into_iter().chain(recent_kept).collect()
2665        } else {
2666            recent_kept
2667        };
2668        let kept_seqs: HashMap<String, i64> = if kept.is_empty() {
2669            HashMap::new()
2670        } else {
2671            let conn = crate::cache::open_meta_db(cache_dir.join(crate::cache::META_DB))?;
2672            let mut stmt = conn.prepare("SELECT walk_seq FROM files WHERE path = ?")?;
2673            let mut out = HashMap::new();
2674            for (p, _) in &kept {
2675                if let Some(seq) = stmt.query_row([p], |r| r.get::<_, i64>(0)).optional()? {
2676                    out.insert(p.clone(), seq);
2677                }
2678            }
2679            out
2680        };
2681        let mut entries: Vec<(i64, String, Source)> = Vec::new();
2682        for (k, r) in read.iter().enumerate() {
2683            if r.is_some() {
2684                entries.push((c.rewrites[k].walk_seq, rels[k].clone(), Source::Fresh(k)));
2685            }
2686        }
2687        for (p, id) in kept {
2688            let seq = kept_seqs.get(&p).copied().unwrap_or(i64::MAX);
2689            entries.push((seq, p, Source::Old(id)));
2690        }
2691        entries.sort_by_key(|(seq, _, _)| *seq);
2692        let mut segment_bytes = 0u64;
2693        for (_, _, src) in &entries {
2694            segment_bytes += match src {
2695                Source::Fresh(k) => read[*k].as_ref().map_or(0, |f| f.content.len() as u64),
2696                Source::Old(id) => snapshot.file_len(*id).unwrap_or(0),
2697            };
2698        }
2699        let (stable_files, stable_bytes) = if fold {
2700            (0, 0)
2701        } else {
2702            (
2703                snapshot.delta_len() as usize
2704                    - new_dead_delta.len()
2705                    - (0..delta_len)
2706                        .filter(|&k| snapshot.is_dead(base_len + k))
2707                        .count(),
2708                delta_kept_bytes,
2709            )
2710        };
2711        if entries.len() + stable_files > max_files || segment_bytes + stable_bytes > max_bytes {
2712            log::info!(
2713                "Delta of {} files / {} bytes passes its limits after reading",
2714                entries.len() + stable_files,
2715                segment_bytes + stable_bytes
2716            );
2717            return Ok(None);
2718        }
2719
2720        // Tombstones, and the planning sizes of the postings they remove.
2721        let mut tombstones: Vec<u32> = if fold {
2722            // A new delta: only base ids stay tombstoned.
2723            prev.tombstones
2724                .iter()
2725                .copied()
2726                .filter(|&id| id < base_len)
2727                .chain(new_dead_base.iter().copied())
2728                .collect()
2729        } else {
2730            prev.tombstones
2731                .iter()
2732                .copied()
2733                .chain(new_dead_base.iter().copied())
2734                .chain(new_dead_delta.iter().copied())
2735                .collect()
2736        };
2737        tombstones.sort_unstable();
2738        tombstones.dedup();
2739        let sizes_of = |ids: &[u32]| -> Vec<Vec<(crate::trigram::Trigram, u32)>> {
2740            c.pool.install(|| {
2741                ids.par_iter()
2742                    .map_init(Vec::<u64>::new, |scratch, &id| {
2743                        let content = snapshot.get_file_content(id).unwrap_or("");
2744                        let run = crate::trigram_build::extract_trigram_run(content, scratch);
2745                        crate::trigram_build::run_plan_sizes(&run)
2746                    })
2747                    .collect()
2748            })
2749        };
2750        let carried = |plan: Option<&crate::snapshot::TombPlan>| {
2751            plan.map(|t| t.entries().map(|(t, n)| (t, n as u64)).collect())
2752                .unwrap_or_default()
2753        };
2754        type TombMap = std::collections::BTreeMap<crate::trigram::Trigram, u64>;
2755        let mut touched_trigrams: Vec<crate::trigram::Trigram> = Vec::new();
2756        let mut tomb: TombMap = carried(snapshot.tomb());
2757        for sizes in sizes_of(&new_dead_base) {
2758            for (t, n) in sizes {
2759                *tomb.entry(t).or_default() += n as u64;
2760                touched_trigrams.push(t);
2761            }
2762        }
2763        let mut tomb_delta: TombMap = if fold {
2764            TombMap::new()
2765        } else {
2766            carried(snapshot.tomb_delta())
2767        };
2768        for sizes in sizes_of(&new_dead_delta) {
2769            for (t, n) in sizes {
2770                if !fold {
2771                    *tomb_delta.entry(t).or_default() += n as u64;
2772                }
2773                touched_trigrams.push(t);
2774            }
2775        }
2776        laps.lap("tombstones");
2777
2778        // Write the new segment (generation files: nothing names them yet).
2779        let (delta_content, delta_trigrams, delta_plan, tomb_name) =
2780            crate::snapshot::delta_file_names(c.generation);
2781        let (recent_content, recent_trigrams, recent_plan, tomb_delta_name) =
2782            crate::snapshot::recent_file_names(c.generation);
2783        let (seg_content, seg_trigrams, seg_plan) = if fold {
2784            (delta_content, delta_trigrams, delta_plan)
2785        } else {
2786            (recent_content, recent_trigrams, recent_plan)
2787        };
2788        let segment = if entries.is_empty() {
2789            None
2790        } else {
2791            let runs: Vec<Option<TrigramRun>> = c.pool.install(|| {
2792                entries
2793                    .par_iter()
2794                    .map_init(Vec::<u64>::new, |scratch, (_, _, src)| match src {
2795                        Source::Fresh(_) => None,
2796                        Source::Old(id) => Some(crate::trigram_build::extract_trigram_run(
2797                            snapshot.get_file_content(*id).unwrap_or(""),
2798                            scratch,
2799                        )),
2800                    })
2801                    .collect()
2802            });
2803            let mut builder = TrigramIndexBuilder::new(cache_dir.join("trigram_temp"));
2804            let mut writer = ContentWriter::new();
2805            let content_path = cache_dir.join(&seg_content);
2806            writer
2807                .init(content_path.clone())
2808                .context("Failed to initialize the delta content store")?;
2809            for ((_, rel, src), run) in entries.iter().zip(runs) {
2810                let path = PathBuf::from(rel);
2811                match src {
2812                    Source::Fresh(k) => {
2813                        let f = read[*k].as_mut().expect("read file");
2814                        let run = std::mem::take(&mut f.trigram_run);
2815                        builder.add_file(path.clone(), run);
2816                        writer.add_file(path, &f.content);
2817                    }
2818                    Source::Old(id) => {
2819                        builder.add_file(path.clone(), run.expect("kept file run"));
2820                        writer.add_file(path, snapshot.get_file_content(*id)?);
2821                    }
2822                }
2823            }
2824            let trigrams_path = cache_dir.join(&seg_trigrams);
2825            builder
2826                .write_with_plan(c.pool, &trigrams_path, Some(&cache_dir.join(&seg_plan)))
2827                .context("Failed to write the delta trigram index")?;
2828            writer
2829                .finalize_if_needed()
2830                .context("Failed to finalize the delta content store")?;
2831            Some(SegmentFiles {
2832                content: seg_content.clone(),
2833                trigrams: seg_trigrams.clone(),
2834                plan: Some(seg_plan.clone()),
2835                files: writer.file_count() as u64,
2836                content_bytes: std::fs::metadata(&content_path)?.len(),
2837                trigrams_bytes: std::fs::metadata(&trigrams_path)?.len(),
2838            })
2839        };
2840        let write_tomb = |map: &TombMap, name: String| -> Result<String> {
2841            let entries: Vec<(crate::trigram::Trigram, u32)> = map
2842                .iter()
2843                .map(|(&t, &n)| (t, n.min(u32::MAX as u64) as u32))
2844                .collect();
2845            crate::snapshot::write_tomb_file(&cache_dir.join(&name), &entries)?;
2846            Ok(name)
2847        };
2848        let tomb_file = if tombstones.iter().any(|&id| id < base_len) {
2849            Some(write_tomb(&tomb, tomb_name)?)
2850        } else {
2851            None
2852        };
2853        let tomb_delta_file = if tombstones.iter().any(|&id| id >= base_len) {
2854            Some(write_tomb(&tomb_delta, tomb_delta_name)?)
2855        } else {
2856            None
2857        };
2858        let (delta_files, recent_files) = if fold {
2859            (segment, None)
2860        } else {
2861            (prev.delta.clone(), segment)
2862        };
2863        laps.lap("write");
2864
2865        // The live trigram count, kept up to date from the trigrams this update
2866        // touches: those of tombstoned files, of the replaced and the new segment.
2867        let segment_index =
2868            |files: &Option<SegmentFiles>| -> Result<Option<crate::trigram::TrigramIndex>> {
2869                let Some(f) = files else { return Ok(None) };
2870                let mut index = crate::trigram::TrigramIndex::load(cache_dir.join(&f.trigrams))?;
2871                if let Some(plan) = &f.plan {
2872                    index.attach_plan(cache_dir.join(plan))?;
2873                }
2874                Ok(Some(index))
2875            };
2876        let new_segment = segment_index(if fold { &delta_files } else { &recent_files })?;
2877        if let Some(recent) = snapshot.recent() {
2878            touched_trigrams.extend(recent.trigrams.trigrams());
2879        }
2880        if fold && let Some(delta) = snapshot.delta() {
2881            touched_trigrams.extend(delta.trigrams.trigrams());
2882        }
2883        if let Some(index) = &new_segment {
2884            touched_trigrams.extend(index.trigrams());
2885        }
2886        touched_trigrams.sort_unstable();
2887        touched_trigrams.dedup();
2888        let plan_in = |index: Option<&crate::trigram::TrigramIndex>, t| -> i64 {
2889            index
2890                .and_then(|i| i.list_part(t))
2891                .map_or(0, |(_, plan)| plan as i64)
2892        };
2893        let old_delta = snapshot.delta().map(|d| &d.trigrams);
2894        let (new_delta, new_recent) = if fold {
2895            (new_segment.as_ref(), None)
2896        } else {
2897            (old_delta, new_segment.as_ref())
2898        };
2899        let mut live_trigrams = prev.live_trigrams as i64;
2900        for &t in &touched_trigrams {
2901            let was = snapshot.live_plan(t) > 0;
2902            let now = plan_in(Some(&snapshot.base().trigrams), t)
2903                + plan_in(new_delta, t)
2904                + plan_in(new_recent, t)
2905                - tomb.get(&t).copied().unwrap_or(0) as i64
2906                - tomb_delta.get(&t).copied().unwrap_or(0) as i64
2907                > 0;
2908            live_trigrams += now as i64 - was as i64;
2909        }
2910        let dead_set: std::collections::HashSet<u32> = tombstones.iter().copied().collect();
2911        // Lengths come from the entry table: reading the content would page in (and
2912        // UTF-8 check) the whole base on every update.
2913        let mut live_corpus = segment_bytes;
2914        let stable_bound = if fold { base_len } else { stable_end };
2915        for id in 0..stable_bound {
2916            if !dead_set.contains(&id) {
2917                live_corpus += snapshot.file_len(id).unwrap_or(0);
2918            }
2919        }
2920        laps.lap("live");
2921
2922        // Publish: fixed names first (an older binary must not read the base alone
2923        // once it is incomplete), then the manifest, then meta.db.
2924        let manifest = Manifest {
2925            format: 0,
2926            generation: c.generation,
2927            base: prev.base.clone(),
2928            delta: delta_files,
2929            recent: recent_files,
2930            tombstones,
2931            tomb: tomb_file,
2932            tomb_delta: tomb_delta_file,
2933            live_trigrams: live_trigrams.max(0) as u64,
2934            publish_id: 0,
2935            live_corpus_bytes: live_corpus,
2936            checksum: String::new(),
2937        }
2938        .sealed();
2939        self.abort_point("delta-files");
2940        if !manifest.base_only() {
2941            crate::snapshot::unlink_fixed_names(&cache_dir);
2942        }
2943        self.abort_point("unlinked");
2944        crate::snapshot::write_manifest(&cache_dir, &manifest)?;
2945        crate::snapshot::link_fixed_names(&cache_dir, &manifest);
2946        self.abort_point("manifest");
2947        log::info!(
2948            "phase delta: {} delta + {} recent files ({} read{}), {} tombstones, {} ms",
2949            manifest.delta.as_ref().map_or(0, |d| d.files),
2950            manifest.recent.as_ref().map_or(0, |d| d.files),
2951            read.iter().filter(|r| r.is_some()).count(),
2952            if fold {
2953                ", folded into a new delta"
2954            } else {
2955                ""
2956            },
2957            manifest.tombstones.len(),
2958            delta_start.elapsed().as_millis()
2959        );
2960        laps.lap("manifest");
2961
2962        // meta.db: the named rows, their dependencies, the statistics and the
2963        // branch row, in one transaction.
2964        let meta_start = Instant::now();
2965        let now = chrono::Utc::now().timestamp();
2966        let flags = (c.dirty)();
2967        laps.lap("dirty");
2968        let mut rows: Vec<crate::cache::FileRow> = Vec::new();
2969        let mut row_k: Vec<usize> = Vec::new();
2970        for (k, r) in read.iter().enumerate() {
2971            if let Some(f) = r {
2972                rows.push(crate::cache::FileRow {
2973                    path: rels[k].clone(),
2974                    hash: f.hash.clone(),
2975                    language: format!("{:?}", f.language),
2976                    line_count: f.line_count,
2977                    size: f.size,
2978                    mtime_ns: f.mtime_ns,
2979                    dirty: flags.rewrites.contains(&rels[k]),
2980                    walk_seq: c.rewrites[k].walk_seq,
2981                });
2982                row_k.push(k);
2983            }
2984        }
2985        let paths_changed =
2986            !deleted.is_empty() || rows.iter().any(|r| !c.stored.contains_key(&r.path));
2987        let mut conn = crate::cache::open_meta_db(cache_dir.join(crate::cache::META_DB))?;
2988        let tx = conn
2989            .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)
2990            .context("Failed to begin meta.db transaction")?;
2991        let deleted_ids: Vec<i64> = deleted.iter().map(|(_, id)| *id).collect();
2992        crate::meta_update::delete_files(&tx, &deleted_ids)?;
2993        let ids = crate::meta_update::upsert_files(&tx, &rows, now)?;
2994        crate::meta_update::set_walk_seqs(&tx, &c.walk_moves)?;
2995        crate::meta_update::refresh_stats(&tx, &flags.touched)?;
2996        crate::meta_update::set_dirty_flags(&tx, &flags.flips)?;
2997        let branch_id = self
2998            .cache
2999            .get_or_create_branch_id(&tx, c.branch, c.commit)?;
3000        if c.sync_all_branch_rows {
3001            crate::meta_update::sync_branch_rows(&tx, branch_id, c.branch, now)?;
3002        } else {
3003            let branch_rows: Vec<(i64, &str)> = ids
3004                .iter()
3005                .zip(&rows)
3006                .map(|(id, row)| (*id, row.hash.as_str()))
3007                .collect();
3008            crate::meta_update::set_branch_rows(&tx, branch_id, &branch_rows, now)?;
3009        }
3010        log::info!(
3011            "meta.db: {} rows written, {} deleted, {} moved, {} touched, {} dirty flags",
3012            rows.len(),
3013            deleted_ids.len(),
3014            c.walk_moves.len(),
3015            flags.touched.len(),
3016            flags.flips.len()
3017        );
3018        laps.lap("rows");
3019
3020        // The resolver this process kept from the previous publish, patched with this
3021        // one's paths, or every path loaded again.
3022        let meta_path = cache_dir.join(crate::cache::META_DB);
3023        let cached = RESOLVER_CACHE
3024            .lock()
3025            .ok()
3026            .and_then(|mut slot| slot.take())
3027            .filter(|(path, publish, _)| {
3028                *path == meta_path && prev.publish_id != 0 && *publish == prev.publish_id
3029            })
3030            .map(|(_, _, resolver)| resolver);
3031        let resolver = match cached {
3032            Some(mut resolver) => {
3033                for (path, _) in &deleted {
3034                    resolver.remove(path);
3035                }
3036                for (row, id) in rows.iter().zip(&ids) {
3037                    resolver.insert(*id, &row.path);
3038                }
3039                resolver
3040            }
3041            None => crate::dependency::PathResolver::from_conn(&tx)
3042                .context("Failed to load file paths for dependency resolution")?,
3043        };
3044        laps.lap("resolver");
3045        let ctx = crate::dependency_resolve::ResolverContext::new(c.root, c.resolver_configs);
3046        let mut writer = crate::dependency::DependencyWriter::new(&tx);
3047        let mut resolved_here: std::collections::HashSet<i64> = std::collections::HashSet::new();
3048        for (&k, &file_id) in row_k.iter().zip(&ids) {
3049            let Some((imports, exports, declared)) =
3050                read[k].as_ref().and_then(|f| f.imports.as_ref())
3051            else {
3052                continue;
3053            };
3054            let rel = &rels[k];
3055            resolved_here.insert(file_id);
3056            let deps = ctx.resolve_file_imports(file_id, rel, imports.clone(), &resolver);
3057            writer.replace_dependencies(file_id, &deps)?;
3058            writer.replace_members(file_id, &ctx.package_members(rel, declared))?;
3059            writer.clear_exports(file_id)?;
3060            for export in exports {
3061                let resolved = ctx.resolve_export(rel, export, &resolver);
3062                writer.insert_export(
3063                    file_id,
3064                    export.exported_symbol.as_deref(),
3065                    &export.source_path,
3066                    resolved,
3067                    export.line_number,
3068                )?;
3069            }
3070        }
3071        let (deps_written, exports_written) = writer.counts();
3072        // An added or removed path can change how an unchanged file's imports
3073        // resolve (suffix matches, ambiguity, the first of several candidates).
3074        let reresolved = if paths_changed {
3075            reresolve(&tx, &ctx, &resolver, &resolved_here)?
3076        } else {
3077            0
3078        };
3079        refresh_vendored(&tx, &c.resolver_configs.vendor, Some(&ids))?;
3080        laps.lap("deps");
3081
3082        crate::meta_update::set_statistic(
3083            &tx,
3084            RESOLVER_DIGEST_KEY,
3085            &c.resolver_configs.digest,
3086            now,
3087        )?;
3088        crate::meta_update::set_statistic(
3089            &tx,
3090            INDEX_GENERATION_KEY,
3091            &c.generation.to_string(),
3092            now,
3093        )?;
3094        CacheManager::update_branch_metadata_on(
3095            &tx,
3096            c.branch,
3097            c.commit,
3098            live_files,
3099            flags.git_dirty,
3100        )?;
3101        // Every file has its branch row now: the count is the file count.
3102        CacheManager::set_total_files_on(&tx, live_files, now)?;
3103        CacheManager::update_schema_hash_on(&tx)?;
3104        CacheManager::update_extraction_hash_on(&tx)?;
3105        laps.lap("stamps");
3106        tx.commit()?;
3107        self.abort_point("meta");
3108        if let Ok(mut slot) = RESOLVER_CACHE.lock() {
3109            *slot = Some((meta_path, manifest.publish_id, resolver));
3110        }
3111        laps.lap("commit");
3112        self.cache
3113            .checkpoint_wal()
3114            .context("Failed to checkpoint WAL")?;
3115        log::info!(
3116            "dependencies: {} rows, {} exports written; {} rows re-resolved",
3117            deps_written,
3118            exports_written,
3119            reresolved
3120        );
3121        log::info!(
3122            "phase meta.db (files, branches, dependencies, exports): {} ms",
3123            meta_start.elapsed().as_millis()
3124        );
3125
3126        crate::query::invalidate_caches(c.root);
3127        drop(snapshot);
3128        crate::snapshot::remove_unreferenced(&cache_dir, &manifest, Some(&prev));
3129        laps.lap("cleanup");
3130
3131        let (mut new_files, mut modified_files, mut unchanged_files) = c.breakdown_rest;
3132        for (k, r) in read.iter().enumerate() {
3133            if let Some(f) = r {
3134                match c.branch_hashes.get(&rels[k]) {
3135                    None => new_files += 1,
3136                    Some(old) if *old != f.hash => modified_files += 1,
3137                    _ => unchanged_files += 1,
3138                }
3139            }
3140        }
3141        let mut stats = if c.full_stats {
3142            // The branch's rows name every file now: count from `files` alone.
3143            self.cache.stats_synced()?
3144        } else {
3145            self.light_stats(live_files, now)
3146        };
3147        laps.lap("stats");
3148        log::info!("delta update phases:{}", laps.line);
3149        stats.new_files = new_files;
3150        stats.modified_files = modified_files;
3151        stats.deleted_files = c.deleted_file_count;
3152        stats.unchanged_files = unchanged_files;
3153        stats.skipped_too_large = c.skipped.0;
3154        stats.skipped_bytes_too_large = c.skipped.1;
3155        stats.skipped_binary = c.skipped.2;
3156        Ok(Some(stats))
3157    }
3158
3159    /// [`Self::process_file`] for a file whose bytes are the ones the published
3160    /// stores hold (`id` there): the text comes from the stores, the hash from its
3161    /// row, and a touched file's stat from the walk. `None` when the stores cannot
3162    /// give it (the caller reads the file).
3163    fn reuse_file(
3164        &self,
3165        ctx: &ProcessCtx<'_>,
3166        snapshot: &crate::snapshot::IndexSnapshot,
3167        id: u32,
3168        i: usize,
3169        meta: &Option<crate::cache::FileStat>,
3170        trigram_scratch: &mut Vec<u64>,
3171    ) -> Option<FileProcessingResult> {
3172        let row = ctx.stored.get(&ctx.rels[i])?;
3173        let content = snapshot.get_file_content(id).ok()?.to_string();
3174        let file_path = &ctx.root.join(&ctx.rels[i]);
3175        let (size, mtime_ns) = match meta {
3176            Some(md) if !row.stat_matches(md) => (md.size(), md.recorded_mtime_ns(ctx.run_start)),
3177            _ => (row.size, row.mtime_ns),
3178        };
3179        let language = Language::from_path(file_path);
3180        let line_count = content.lines().count();
3181        let trigram_run = crate::trigram_build::extract_trigram_run(&content, trigram_scratch);
3182        let imports = ctx.full_deps.then(|| {
3183            let path_str = file_path.to_string_lossy().to_string();
3184            extract_imports(language, &content, &path_str, ctx.root, ctx.tsconfigs)
3185        });
3186        Some(FileProcessingResult {
3187            hash: row.hash.clone(),
3188            content,
3189            language,
3190            line_count,
3191            size,
3192            mtime_ns,
3193            imports,
3194            trigram_run,
3195        })
3196    }
3197
3198    /// Read one discovered file: stat, bytes, hash, text, trigram run, and imports
3199    /// when they may have changed. `None` when it cannot be read.
3200    fn process_file(
3201        &self,
3202        ctx: &ProcessCtx<'_>,
3203        i: usize,
3204        trigram_scratch: &mut Vec<u64>,
3205    ) -> Option<FileProcessingResult> {
3206        let file_path = &ctx.root.join(&ctx.rels[i]);
3207        let path_str = file_path.to_string_lossy().to_string();
3208
3209        // Stat BEFORE the read. If the file changes between the two, the recorded
3210        // (size, mtime) is older than the bytes, so the next check re-hashes it
3211        // rather than trusting a stat that matches.
3212        let (size, mtime_ns) = std::fs::metadata(file_path)
3213            .map(|md| {
3214                (
3215                    md.len(),
3216                    crate::cache::recorded_mtime_ns(&md, ctx.run_start),
3217                )
3218            })
3219            .unwrap_or((0, 0));
3220
3221        // Read file content once (used for hashing, trigrams, and parsing). The
3222        // hash is of the RAW bytes, so the freshness check can hash a file on disk
3223        // and compare. Invalid UTF-8 (a Latin-1 `.po`, an old doc) is decoded
3224        // lossily rather than dropped: ripgrep searches those bytes, and an agent
3225        // expects the same.
3226        let bytes = match std::fs::read(file_path) {
3227            Ok(b) => b,
3228            Err(e) => {
3229                log::warn!("Failed to read {}: {}", path_str, e);
3230                return None;
3231            }
3232        };
3233        let hash = self.hash_content(&bytes);
3234        let content = match String::from_utf8(bytes) {
3235            Ok(s) => s,
3236            Err(e) => String::from_utf8_lossy(e.as_bytes()).into_owned(),
3237        };
3238
3239        // Detect language
3240        let language = Language::from_path(file_path);
3241
3242        // Count lines in the file
3243        let line_count = content.lines().count();
3244
3245        // Trigram postings, sorted, without a file id (assigned serially).
3246        let trigram_run = crate::trigram_build::extract_trigram_run(&content, trigram_scratch);
3247
3248        // Imports and re-exports: for every file when they are all rewritten, else
3249        // for files whose bytes changed (including a file that changed again since
3250        // it was classified).
3251        let unchanged = ctx
3252            .stored
3253            .get(&ctx.rels[i])
3254            .is_some_and(|row| row.hash == hash);
3255        let imports = (ctx.full_deps || !unchanged)
3256            .then(|| extract_imports(language, &content, &path_str, ctx.root, ctx.tsconfigs));
3257
3258        Some(FileProcessingResult {
3259            hash,
3260            content,
3261            language,
3262            line_count,
3263            size,
3264            mtime_ns,
3265            imports,
3266            trigram_run,
3267        })
3268    }
3269
3270    /// Write the marker whose mtime is this run's race threshold; returns it (or the
3271    /// process clock when the marker cannot be written or read).
3272    fn run_marker(&self, cache_dir: &Path) -> std::time::SystemTime {
3273        let marker = cache_dir.join(RUN_MARKER);
3274        std::fs::write(&marker, b"")
3275            .and_then(|_| std::fs::metadata(&marker))
3276            .and_then(|md| md.modified())
3277            .unwrap_or_else(|_| std::time::SystemTime::now())
3278    }
3279
3280    /// Whether the published snapshot is complete, holds `expected` files (the rows
3281    /// of the last indexed tree) and is the one meta.db's rows go with.
3282    fn stores_intact(&self, expected: usize, meta_generation: Option<u64>) -> bool {
3283        // A manifest, headers and sizes: not the stores themselves (a run that
3284        // changes nothing must not page in their path tables). Whatever opens them
3285        // next validates them in full.
3286        match crate::snapshot::check_published(self.cache.path()) {
3287            Ok(None) => {
3288                log::info!("Index written before the manifest - rebuilding");
3289                false
3290            }
3291            Ok(Some(manifest)) => {
3292                if Some(manifest.generation) != meta_generation {
3293                    log::warn!(
3294                        "Manifest generation {} but meta.db rows are generation {:?} (a run stopped between the two) - rebuilding",
3295                        manifest.generation,
3296                        meta_generation
3297                    );
3298                    false
3299                } else if manifest.live_files() as usize != expected {
3300                    log::warn!(
3301                        "Stores hold {} files but meta.db lists {} - rebuilding",
3302                        manifest.live_files(),
3303                        expected
3304                    );
3305                    false
3306                } else {
3307                    true
3308                }
3309            }
3310            Err(e) => {
3311                log::warn!("Index stores unusable ({:#}) - rebuilding", e);
3312                false
3313            }
3314        }
3315    }
3316
3317    /// Nothing to write to the stores: bring meta.db up to date (fingerprints of
3318    /// touched files, dirty flags, walk positions, this branch's rows, the branch
3319    /// metadata and the statistics timestamp) and report.
3320    /// The statistics of a library update: counts and sizes, without the
3321    /// per-language maps (they scan every row).
3322    fn light_stats(&self, total_files: usize, now: i64) -> IndexStats {
3323        let (index_size_bytes, trigram_index_bytes, corpus_bytes) = self.cache.store_sizes();
3324        IndexStats {
3325            total_files,
3326            index_size_bytes,
3327            last_updated: chrono::DateTime::from_timestamp(now, 0)
3328                .unwrap_or_else(chrono::Utc::now)
3329                .to_rfc3339(),
3330            corpus_bytes,
3331            trigram_index_bytes,
3332            ..Default::default()
3333        }
3334    }
3335
3336    /// A library update that changed no content: the named rows' stats, flags and
3337    /// positions, the branch row and the statistics (what `refresh_unchanged`
3338    /// does for a whole walk).
3339    fn refresh_named(&self, r: RefreshNamed<'_>) -> Result<IndexStats> {
3340        let mut conn = crate::cache::open_meta_db(self.cache.path().join(crate::cache::META_DB))?;
3341        let tx = conn
3342            .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)
3343            .context("Failed to begin meta.db transaction")?;
3344        crate::meta_update::set_walk_seqs(&tx, r.walk_moves)?;
3345        crate::meta_update::refresh_stats(&tx, r.touched)?;
3346        crate::meta_update::set_dirty_flags(&tx, r.flips)?;
3347        if r.commit.is_some() {
3348            CacheManager::update_branch_metadata_on(
3349                &tx,
3350                r.branch,
3351                r.commit,
3352                r.live_files,
3353                r.git_dirty,
3354            )?;
3355        }
3356        CacheManager::update_stats_on(&tx, r.branch)?;
3357        tx.commit()?;
3358        let mut stats = self.light_stats(r.live_files, chrono::Utc::now().timestamp());
3359        stats.new_files = 0;
3360        stats.modified_files = 0;
3361        stats.deleted_files = 0;
3362        stats.unchanged_files = r.live_files;
3363        stats.skipped_too_large = r.skipped.0;
3364        stats.skipped_bytes_too_large = r.skipped.1;
3365        stats.skipped_binary = r.skipped.2;
3366        Ok(stats)
3367    }
3368
3369    fn refresh_unchanged(&self, r: RefreshUnchanged<'_>) -> Result<IndexStats> {
3370        let mut laps = Laps::new();
3371        let now = chrono::Utc::now().timestamp();
3372        let rows: Vec<&crate::meta_update::StoredFile> = r
3373            .rows
3374            .iter()
3375            .map(|row| row.expect("nothing was added: every file has a row"))
3376            .collect();
3377        let seqs = crate::meta_update::plan_walk_seq(
3378            &rows
3379                .iter()
3380                .map(|row| Some(row.walk_seq))
3381                .collect::<Vec<_>>(),
3382        );
3383        let mut walk = Vec::new();
3384        let mut flips = Vec::new();
3385        let mut touched = Vec::new();
3386        let no_dirty = r.dirty_paths.is_empty();
3387        for (i, rel) in r.rels.iter().enumerate() {
3388            let row = rows[i];
3389            let dirty = !no_dirty && r.dirty_paths.contains(rel);
3390            if seqs[i] != row.walk_seq {
3391                walk.push((row.id, seqs[i]));
3392            }
3393            if r.status[i] == FileStatus::Touched {
3394                let (size, mtime) = r.metas[i]
3395                    .as_ref()
3396                    .map(|md| (md.size(), md.recorded_mtime_ns(r.run_start)))
3397                    .unwrap_or((0, 0));
3398                touched.push((row.id, size, mtime, dirty));
3399            } else if dirty != row.dirty {
3400                flips.push((row.id, dirty));
3401            }
3402        }
3403
3404        laps.lap("plan");
3405        let mut conn = crate::cache::open_meta_db(self.cache.path().join(crate::cache::META_DB))?;
3406        let tx = conn
3407            .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)
3408            .context("Failed to begin meta.db transaction")?;
3409        crate::meta_update::set_walk_seqs(&tx, &walk)?;
3410        crate::meta_update::refresh_stats(&tx, &touched)?;
3411        crate::meta_update::set_dirty_flags(&tx, &flips)?;
3412        let branch_id = self
3413            .cache
3414            .get_or_create_branch_id(&tx, r.branch, r.commit)?;
3415        laps.lap("rows");
3416        if !r.in_sync {
3417            crate::meta_update::sync_branch_rows(&tx, branch_id, r.branch, now)?;
3418        }
3419        laps.lap("branch_rows");
3420
3421        // The CONTENT is current, but the recorded commit may not be: committing
3422        // already-indexed files moves HEAD without changing a single hash.
3423        // Skipping the metadata update left `commit_sha` behind forever, so
3424        // freshness reported `stale` on a perfectly current index until some
3425        // unrelated edit happened to force a rebuild.
3426        if let Some(git_dirty) = r.git_dirty {
3427            CacheManager::update_branch_metadata_on(
3428                &tx,
3429                r.branch,
3430                r.commit,
3431                r.rels.len(),
3432                git_dirty,
3433            )?;
3434        }
3435        // Every file has its branch row now: the count is the file count.
3436        CacheManager::set_total_files_on(&tx, r.rels.len(), now)?;
3437        tx.commit()?;
3438        laps.lap("commit");
3439
3440        // On the synced branch every file's branch hash is its row's: all unchanged.
3441        let (new_files, modified_files, unchanged_files) = if r.in_sync {
3442            (0, 0, rows.len())
3443        } else {
3444            breakdown(
3445                r.rels
3446                    .iter()
3447                    .zip(&rows)
3448                    .map(|(rel, row)| (rel.as_str(), row.hash.as_str())),
3449                r.existing_hashes,
3450            )
3451        };
3452        // Nothing changed: the statistics are the rows' (what `stats_synced` would
3453        // count from `files`), without a query.
3454        let mut files_by_language: HashMap<String, usize> = HashMap::new();
3455        let mut lines_by_language: HashMap<String, usize> = HashMap::new();
3456        for row in &rows {
3457            match files_by_language.get_mut(&row.language) {
3458                Some(n) => *n += 1,
3459                None => {
3460                    files_by_language.insert(row.language.clone(), 1);
3461                }
3462            }
3463            match lines_by_language.get_mut(&row.language) {
3464                Some(n) => *n += row.line_count,
3465                None => {
3466                    lines_by_language.insert(row.language.clone(), row.line_count);
3467                }
3468            }
3469        }
3470        let (index_size_bytes, trigram_index_bytes, corpus_bytes) = self.cache.store_sizes();
3471        let mut stats = IndexStats {
3472            total_files: r.rels.len(),
3473            index_size_bytes,
3474            last_updated: chrono::DateTime::from_timestamp(now, 0)
3475                .unwrap_or_else(chrono::Utc::now)
3476                .to_rfc3339(),
3477            files_by_language,
3478            lines_by_language,
3479            corpus_bytes,
3480            trigram_index_bytes,
3481            ..Default::default()
3482        };
3483        laps.lap("stats");
3484        log::info!("refresh phases:{}", laps.line);
3485        stats.new_files = new_files;
3486        stats.modified_files = modified_files;
3487        stats.unchanged_files = unchanged_files;
3488        stats.deleted_files = 0;
3489        stats.skipped_too_large = r.skipped.0;
3490        stats.skipped_bytes_too_large = r.skipped.1;
3491        stats.skipped_binary = r.skipped.2;
3492        Ok(stats)
3493    }
3494
3495    /// One meta.db transaction for a run that rewrote the stores: the rows of
3496    /// files whose bytes, fingerprint, dirty flag or walk position changed, the
3497    /// deletions, this branch's rows, and the dependency and export rows (every
3498    /// file's when `full_deps`, else the changed files', plus re-resolution of the
3499    /// rest when files were added or removed).
3500    fn write_meta(&self, w: MetaWrite<'_>) -> Result<()> {
3501        let now = chrono::Utc::now().timestamp();
3502        let mut laps = Laps::new();
3503        let seqs = crate::meta_update::plan_walk_seq(
3504            &w.present
3505                .iter()
3506                .map(|&i| w.stored.get(&w.rels[i]).map(|s| s.walk_seq))
3507                .collect::<Vec<_>>(),
3508        );
3509        let written: HashMap<usize, &IndexedFile> =
3510            w.written.iter().map(|f| (f.index, f)).collect();
3511
3512        let mut rows: Vec<crate::cache::FileRow> = Vec::new();
3513        let mut row_files: Vec<usize> = Vec::new(); // discovery index of each row
3514        let mut walk = Vec::new();
3515        let mut flips = Vec::new();
3516        let mut touched = Vec::new();
3517        for (k, &i) in w.present.iter().enumerate() {
3518            let rel = &w.rels[i];
3519            let dirty = w.dirty_paths.contains(rel);
3520            let stored = w.stored.get(rel);
3521            match written.get(&i) {
3522                Some(f)
3523                    if !(w.status[i] == FileStatus::Unchanged
3524                        && stored.is_some_and(|row| row.hash == f.hash)) =>
3525                {
3526                    rows.push(crate::cache::FileRow {
3527                        path: rel.clone(),
3528                        hash: f.hash.clone(),
3529                        language: format!("{:?}", f.language),
3530                        line_count: f.line_count,
3531                        size: f.size,
3532                        mtime_ns: f.mtime_ns,
3533                        dirty,
3534                        walk_seq: seqs[k],
3535                    });
3536                    row_files.push(i);
3537                }
3538                _ => {
3539                    // Not read this run (or read with the bytes and fingerprint
3540                    // stored): only the walk position, the fingerprint of a touched
3541                    // file and the dirty flag can move.
3542                    let Some(row) = stored else { continue };
3543                    if seqs[k] != row.walk_seq {
3544                        walk.push((row.id, seqs[k]));
3545                    }
3546                    if w.status[i] == FileStatus::Touched && !written.contains_key(&i) {
3547                        let (size, mtime) = w.metas[i]
3548                            .as_ref()
3549                            .map(|md| (md.size(), md.recorded_mtime_ns(w.run_start)))
3550                            .unwrap_or((0, 0));
3551                        touched.push((row.id, size, mtime, dirty));
3552                    } else if dirty != row.dirty {
3553                        flips.push((row.id, dirty));
3554                    }
3555                }
3556            }
3557        }
3558        let present: std::collections::HashSet<&str> =
3559            w.present.iter().map(|&i| w.rels[i].as_str()).collect();
3560        let deleted: Vec<i64> = w
3561            .stored
3562            .iter()
3563            .filter(|(p, _)| !present.contains(p.as_str()))
3564            .map(|(_, row)| row.id)
3565            .collect();
3566        let paths_changed = !deleted.is_empty()
3567            || w.present
3568                .iter()
3569                .any(|&i| !w.stored.contains_key(&w.rels[i]));
3570
3571        laps.lap("plan");
3572        let mut conn = crate::cache::open_meta_db(self.cache.path().join(crate::cache::META_DB))?;
3573        let tx = conn
3574            .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)
3575            .context("Failed to begin meta.db transaction")?;
3576
3577        crate::meta_update::delete_files(&tx, &deleted)?;
3578        let new_ids = crate::meta_update::upsert_files(&tx, &rows, now)?;
3579        crate::meta_update::set_walk_seqs(&tx, &walk)?;
3580        crate::meta_update::refresh_stats(&tx, &touched)?;
3581        crate::meta_update::set_dirty_flags(&tx, &flips)?;
3582        laps.lap("rows");
3583        let branch_id = self
3584            .cache
3585            .get_or_create_branch_id(&tx, w.branch, w.commit)?;
3586        crate::meta_update::sync_branch_rows(&tx, branch_id, w.branch, now)?;
3587        laps.lap("branch_rows");
3588        log::info!(
3589            "meta.db: {} rows written, {} deleted, {} moved, {} touched, {} dirty flags",
3590            rows.len(),
3591            deleted.len(),
3592            walk.len(),
3593            touched.len(),
3594            flips.len()
3595        );
3596
3597        // id of every file read this run, by discovery index.
3598        let mut ids: HashMap<usize, i64> = w
3599            .written
3600            .iter()
3601            .filter_map(|f| w.stored.get(&w.rels[f.index]).map(|s| (f.index, s.id)))
3602            .collect();
3603        for (i, id) in row_files.iter().zip(new_ids) {
3604            ids.insert(*i, id);
3605        }
3606
3607        let resolver = crate::dependency::PathResolver::from_conn(&tx)
3608            .context("Failed to load file paths for dependency resolution")?;
3609        laps.lap("resolver");
3610        let ctx = crate::dependency_resolve::ResolverContext::new(w.root, w.resolver_configs);
3611        let mut writer = crate::dependency::DependencyWriter::new(&tx);
3612        if w.full_deps {
3613            writer.clear_all()?;
3614        }
3615        // In walk order: the row order a full build produces.
3616        let mut resolved_here: std::collections::HashSet<i64> = std::collections::HashSet::new();
3617        for f in w.written {
3618            let Some((imports, exports, declared)) = &f.imports else {
3619                continue;
3620            };
3621            let rel = &w.rels[f.index];
3622            let file_id = ids[&f.index];
3623            resolved_here.insert(file_id);
3624            let deps = ctx.resolve_file_imports(file_id, rel, imports.clone(), &resolver);
3625            writer.replace_dependencies(file_id, &deps)?;
3626            writer.replace_members(file_id, &ctx.package_members(rel, declared))?;
3627            if !w.full_deps {
3628                writer.clear_exports(file_id)?;
3629            }
3630            for export in exports {
3631                let resolved = ctx.resolve_export(rel, export, &resolver);
3632                writer.insert_export(
3633                    file_id,
3634                    export.exported_symbol.as_deref(),
3635                    &export.source_path,
3636                    resolved,
3637                    export.line_number,
3638                )?;
3639            }
3640        }
3641        let (deps_written, exports_written) = writer.counts();
3642        laps.lap("deps");
3643
3644        // An added or removed path can change how an unchanged file's imports
3645        // resolve (suffix matches, ambiguity, the first of several candidates).
3646        let mut reresolved = 0usize;
3647        if !w.full_deps && paths_changed {
3648            reresolved = reresolve(&tx, &ctx, &resolver, &resolved_here)?;
3649        }
3650        let written_ids: Vec<i64> = ids.values().copied().collect();
3651        refresh_vendored(
3652            &tx,
3653            &w.resolver_configs.vendor,
3654            (!w.full_deps).then_some(written_ids.as_slice()),
3655        )?;
3656
3657        crate::meta_update::set_statistic(
3658            &tx,
3659            RESOLVER_DIGEST_KEY,
3660            &w.resolver_configs.digest,
3661            now,
3662        )?;
3663        crate::meta_update::set_statistic(
3664            &tx,
3665            INDEX_GENERATION_KEY,
3666            &w.generation.to_string(),
3667            now,
3668        )?;
3669        laps.lap("reresolve");
3670        tx.commit()?;
3671        laps.lap("commit");
3672        log::info!("meta.db write phases:{}", laps.line);
3673        log::info!(
3674            "dependencies: {} rows, {} exports written; {} rows re-resolved",
3675            deps_written,
3676            exports_written,
3677            reresolved
3678        );
3679        Ok(())
3680    }
3681
3682    /// Discover all indexable files in the directory tree.
3683    ///
3684    /// `stored` is meta.db's view of the last indexed tree: a non-code file whose
3685    /// stat matches its row was sniffed for NUL bytes when it was indexed and is
3686    /// not read again.
3687    #[cfg(test)]
3688    fn discover_files(
3689        &self,
3690        root: &Path,
3691        stored: &HashMap<String, crate::meta_update::StoredFile>,
3692    ) -> Result<Discovered> {
3693        self.discover_files_in(root, stored, None)
3694    }
3695
3696    /// [`Self::discover_files`] limited to `targets` (all files when `None`): the
3697    /// same walker and rules, entering only the named paths and their ancestors, so
3698    /// every ignore file on the way applies exactly as in a full walk.
3699    fn discover_files_in(
3700        &self,
3701        root: &Path,
3702        stored: &HashMap<String, crate::meta_update::StoredFile>,
3703        targets: Option<Arc<Targets>>,
3704    ) -> Result<Discovered> {
3705        let walked = self.walk_candidates(root, targets)?;
3706        Ok(Self::drop_binaries(root, walked, stored, None))
3707    }
3708
3709    /// The walk half of discovery: every file the walker and the path policy admit,
3710    /// under the size limit, with its stat, in walk order. Non-code files are only
3711    /// marked: [`Self::drop_binaries`] decides, once the stored rows are known.
3712    fn walk_candidates(&self, root: &Path, targets: Option<Arc<Targets>>) -> Result<Walked> {
3713        let mut out = Walked::default();
3714
3715        let policy = self.path_policy(root);
3716        let walker = Self::targeted_walk_builder(root, &self.config, &policy, targets).build();
3717
3718        for entry in walker {
3719            let entry = entry?;
3720            let path = entry.path();
3721
3722            // Only process files (not directories)
3723            if !entry.file_type().map(|ft| ft.is_file()).unwrap_or(false) {
3724                continue;
3725            }
3726
3727            // Check extension / language eligibility first (cheap)
3728            let Some(lang) = policy.classify(path) else {
3729                continue;
3730            };
3731
3732            // Check file size separately so we can report skipped counts
3733            let mut size = 0u64;
3734            let metadata = std::fs::metadata(path).ok();
3735            if let Some(metadata) = &metadata {
3736                size = metadata.len();
3737                if size > self.config.max_file_size as u64 {
3738                    log::debug!("Skipping {} (too large: {} bytes)", path.display(), size);
3739                    out.found.skipped_too_large += 1;
3740                    out.found.skipped_bytes_too_large += size;
3741                    continue;
3742                }
3743            }
3744
3745            out.found.rels.push(normalize_rel(root, path));
3746            out.found.sizes.push(size);
3747            out.found
3748                .metas
3749                .push(metadata.as_ref().map(crate::cache::FileStat::of));
3750            out.non_code.push(!lang.is_code());
3751        }
3752
3753        Ok(out)
3754    }
3755
3756    /// The binary half of discovery. A code extension is trusted to be text.
3757    /// Anything else in the tracked tier (`image.png`, `OWNERS`, `data.bin`) is
3758    /// sniffed: ripgrep's rule, a NUL byte anywhere means binary, and a binary file
3759    /// is never in the index. Only the long tail pays the read (from the page
3760    /// cache, since the main pass reads it again a moment later), and not a file
3761    /// that is unchanged since it was indexed. Order is kept.
3762    fn drop_binaries(
3763        root: &Path,
3764        walked: Walked,
3765        stored: &HashMap<String, crate::meta_update::StoredFile>,
3766        pool: Option<&rayon::ThreadPool>,
3767    ) -> Discovered {
3768        let Walked {
3769            found: mut out,
3770            non_code,
3771        } = walked;
3772        let to_sniff: Vec<usize> = (0..out.rels.len())
3773            .filter(|&i| {
3774                non_code[i]
3775                    && !stored
3776                        .get(&out.rels[i])
3777                        .zip(out.metas[i].as_ref())
3778                        .is_some_and(|(row, md)| row.stat_matches(md))
3779            })
3780            .collect();
3781        let sniff = |i: &usize| looks_binary(&root.join(&out.rels[*i]));
3782        let binary: Vec<bool> = match pool {
3783            Some(pool) => pool.install(|| to_sniff.par_iter().map(sniff).collect()),
3784            None => to_sniff.iter().map(sniff).collect(),
3785        };
3786        let dropped: std::collections::HashSet<usize> = to_sniff
3787            .iter()
3788            .zip(binary)
3789            .filter_map(|(&i, b)| b.then_some(i))
3790            .collect();
3791        if dropped.is_empty() {
3792            return out;
3793        }
3794        for &i in &dropped {
3795            log::debug!("Skipping {} (binary)", out.rels[i]);
3796        }
3797        out.skipped_binary += dropped.len();
3798        fn keep<T>(v: &mut Vec<T>, dropped: &std::collections::HashSet<usize>) {
3799            let mut i = 0;
3800            v.retain(|_| {
3801                let k = !dropped.contains(&i);
3802                i += 1;
3803                k
3804            });
3805        }
3806        keep(&mut out.rels, &dropped);
3807        keep(&mut out.sizes, &dropped);
3808        keep(&mut out.metas, &dropped);
3809        out
3810    }
3811
3812    /// The directory walker every tree pass shares: the indexer, and the freshness
3813    /// check outside git (which has no `git status` to name candidates and must
3814    /// walk). One builder so the two can never disagree about what is in the tree.
3815    /// [`Indexer::walk_builder`], entering only `targets` and their ancestors when
3816    /// given.
3817    fn targeted_walk_builder(
3818        root: &Path,
3819        config: &IndexConfig,
3820        policy: &PathPolicy,
3821        targets: Option<Arc<Targets>>,
3822    ) -> WalkBuilder {
3823        let mut builder = Self::walk_builder(root, config, policy);
3824        if let Some(targets) = targets {
3825            // Replaces the walker's own entry filter, so repeat it.
3826            let hidden = policy.hidden();
3827            let root_buf = root.to_path_buf();
3828            builder.filter_entry(move |e| {
3829                if hidden {
3830                    let name = e.file_name();
3831                    if name == ".git" || name == crate::cache::CACHE_DIR {
3832                        return false;
3833                    }
3834                }
3835                let rel = normalize_rel(&root_buf, e.path());
3836                targets.admits(&rel)
3837            });
3838        }
3839        builder
3840    }
3841
3842    /// Which of the files `rels` (relative, `/`-separated) the walk reaches: the
3843    /// ignore files and the include/exclude patterns applied exactly as an index
3844    /// run applies them. Walks only the ancestors of `rels`.
3845    pub(crate) fn walked_among(
3846        root: &Path,
3847        config: &IndexConfig,
3848        policy: &PathPolicy,
3849        rels: &[String],
3850    ) -> std::collections::HashSet<String> {
3851        let targets = Arc::new(Targets::new(rels));
3852        let walker =
3853            Self::targeted_walk_builder(root, config, policy, Some(Arc::clone(&targets))).build();
3854        walker
3855            .flatten()
3856            .filter(|e| e.file_type().is_some_and(|t| t.is_file()))
3857            .map(|e| normalize_rel(root, e.path()))
3858            .filter(|rel| targets.exact.contains(rel))
3859            .collect()
3860    }
3861
3862    pub fn walk_builder(root: &Path, config: &IndexConfig, policy: &PathPolicy) -> WalkBuilder {
3863        // WalkBuilder from ignore crate automatically respects:
3864        // - .gitignore (when in a git repo)
3865        // - .ignore files
3866        // - Hidden files (can be configured)
3867        let mut builder = WalkBuilder::new(root);
3868        builder
3869            .follow_links(config.follow_symlinks)
3870            .hidden(!policy.hidden())
3871            .git_ignore(true) // Explicitly enable gitignore support (enabled by default, but be explicit)
3872            .git_global(false) // Don't use global gitignore
3873            .git_exclude(false); // Don't use .git/info/exclude
3874        if policy.hidden() {
3875            // Dot-directories are walked, but never the repository's own and never
3876            // Reflex's own cache: indexing `.reflex/content.bin` into content.bin
3877            // is a loop, and `.git/objects` is binary noise by the thousand.
3878            builder.filter_entry(|e| {
3879                let name = e.file_name();
3880                name != ".git" && name != crate::cache::CACHE_DIR
3881            });
3882        }
3883        // `[index] include.patterns` / `exclude.patterns`, gitignore semantics.
3884        if let Some(ov) = policy.overrides() {
3885            builder.overrides(ov.clone());
3886        }
3887        builder
3888    }
3889
3890    /// Whether a path is one Reflex would index, judged by extension alone.
3891    ///
3892    /// Public so freshness checking can ask the same question the walker asks. If the
3893    /// two ever disagree, editing a file Reflex does not index (a README, anything
3894    /// under `target/`) would mark the index permanently stale — a cure worse than
3895    /// the disease.
3896    ///
3897    /// Judges extension only: no filesystem access, no size check, no `.gitignore`
3898    /// (git's own output is already filtered by that). Cheap enough to call per path
3899    /// in a `git status` listing.
3900    pub fn is_indexable_path(path: &Path) -> bool {
3901        Self::is_indexable_path_with(path, None)
3902    }
3903
3904    /// [`Self::is_indexable_path`] under an `[index]` policy.
3905    ///
3906    /// The freshness check must apply the same policy as the walker: a file the
3907    /// config excludes is not indexed, so editing it must not report staleness.
3908    pub fn is_indexable_path_with(path: &Path, policy: Option<&PathPolicy>) -> bool {
3909        let default_policy;
3910        let policy = match policy {
3911            Some(p) => p,
3912            None => {
3913                default_policy = PathPolicy::default();
3914                &default_policy
3915            }
3916        };
3917        // Mirror the walker's hidden rule. Without this, Reflex's OWN
3918        // `.reflex/config.toml` counts as an indexable change the moment the text
3919        // tier claims `.toml`, and the index reports itself permanently stale.
3920        policy.hidden_ok(path) && policy.classify(path).is_some()
3921    }
3922
3923    /// Check if a file's language/extension is eligible for indexing (without size check).
3924    fn should_index_lang(&self, path: &Path) -> bool {
3925        PathPolicy::from_config(Path::new("."), &self.config)
3926            .classify(path)
3927            .is_some()
3928    }
3929
3930    /// Check if a file should be indexed based on config (language + size).
3931    #[allow(dead_code)]
3932    fn should_index(&self, path: &Path) -> bool {
3933        if !self.should_index_lang(path) {
3934            return false;
3935        }
3936
3937        // Check file size limits
3938        if let Ok(metadata) = std::fs::metadata(path)
3939            && metadata.len() > self.config.max_file_size as u64
3940        {
3941            log::debug!(
3942                "Skipping {} (too large: {} bytes)",
3943                path.display(),
3944                metadata.len()
3945            );
3946            return false;
3947        }
3948
3949        // `[index] include/exclude` patterns are applied by the walker
3950        // (`discover_files`) and by `is_indexable_path_with`; this per-file check
3951        // has no root to anchor them against.
3952        true
3953    }
3954
3955    /// Compute blake3 hash from file contents for change detection
3956    fn hash_content(&self, content: &[u8]) -> String {
3957        let hash = blake3::hash(content);
3958        hash.to_hex().to_string()
3959    }
3960
3961    /// Check available disk space before indexing
3962    ///
3963    /// Ensures there's enough free space to create the index. Warns if disk space is low.
3964    /// This prevents partial index writes and confusing error messages.
3965    #[cfg_attr(not(unix), allow(unused_variables))]
3966    fn check_disk_space(&self, root: &Path) -> Result<()> {
3967        // Get available space on the filesystem containing the cache directory
3968        let cache_path = self.cache.path();
3969
3970        // Use statvfs on Unix systems
3971        #[cfg(unix)]
3972        {
3973            // On Linux, we can use statvfs to get available space
3974            // For now, we'll use a simple heuristic: warn if we can't write a test file
3975            let test_file = cache_path.join(".space_check");
3976            match std::fs::write(&test_file, b"test") {
3977                Ok(_) => {
3978                    let _ = std::fs::remove_file(&test_file);
3979
3980                    // Try to estimate available space using df command
3981                    if let Ok(output) = std::process::Command::new("df")
3982                        .arg("-k")
3983                        .arg(cache_path.parent().unwrap_or(root))
3984                        .output()
3985                        && let Ok(df_output) = String::from_utf8(output.stdout)
3986                    {
3987                        // Parse df output to get available KB
3988                        if let Some(line) = df_output.lines().nth(1) {
3989                            let parts: Vec<&str> = line.split_whitespace().collect();
3990                            if parts.len() >= 4
3991                                && let Ok(available_kb) = parts[3].parse::<u64>()
3992                            {
3993                                let available_mb = available_kb / 1024;
3994
3995                                // Warn if less than 100MB available
3996                                if available_mb < 100 {
3997                                    log::warn!(
3998                                        "Low disk space: only {}MB available. Indexing may fail.",
3999                                        available_mb
4000                                    );
4001                                    output::warn(&format!(
4002                                        "Low disk space ({}MB available). Consider freeing up space.",
4003                                        available_mb
4004                                    ));
4005                                } else {
4006                                    log::debug!("Available disk space: {}MB", available_mb);
4007                                }
4008                            }
4009                        }
4010                    }
4011
4012                    Ok(())
4013                }
4014                Err(e) if e.kind() == std::io::ErrorKind::PermissionDenied => {
4015                    anyhow::bail!(
4016                        "Permission denied writing to cache directory: {}. Check file permissions.",
4017                        cache_path.display()
4018                    )
4019                }
4020                Err(e) => {
4021                    // If we can't write, it might be a disk space issue
4022                    log::warn!(
4023                        "Failed to write test file (possible disk space issue): {}",
4024                        e
4025                    );
4026                    Err(e).context(
4027                        "Failed to verify disk space - indexing may fail due to insufficient space",
4028                    )
4029                }
4030            }
4031        }
4032
4033        #[cfg(not(unix))]
4034        {
4035            // On Windows, try to write a test file
4036            let test_file = cache_path.join(".space_check");
4037            match std::fs::write(&test_file, b"test") {
4038                Ok(_) => {
4039                    let _ = std::fs::remove_file(&test_file);
4040                    Ok(())
4041                }
4042                Err(e) if e.kind() == std::io::ErrorKind::PermissionDenied => {
4043                    anyhow::bail!(
4044                        "Permission denied writing to cache directory: {}. Check file permissions.",
4045                        cache_path.display()
4046                    )
4047                }
4048                Err(e) => {
4049                    log::warn!(
4050                        "Failed to write test file (possible disk space issue): {}",
4051                        e
4052                    );
4053                    Err(e).context(
4054                        "Failed to verify disk space - indexing may fail due to insufficient space",
4055                    )
4056                }
4057            }
4058        }
4059    }
4060}
4061
4062#[cfg(test)]
4063mod tests {
4064    use super::*;
4065    use std::fs;
4066    use tempfile::TempDir;
4067
4068    #[test]
4069    fn test_indexer_creation() {
4070        let temp = TempDir::new().unwrap();
4071        let cache = CacheManager::new(temp.path());
4072        let config = IndexConfig::default();
4073        let indexer = Indexer::new(cache, config);
4074
4075        assert!(indexer.cache.path().ends_with(".reflex"));
4076    }
4077
4078    #[test]
4079    fn test_hash_content() {
4080        let temp = TempDir::new().unwrap();
4081        let cache = CacheManager::new(temp.path());
4082        let config = IndexConfig::default();
4083        let indexer = Indexer::new(cache, config);
4084
4085        let content1 = b"hello world";
4086        let content2 = b"hello world";
4087        let content3 = b"different content";
4088
4089        let hash1 = indexer.hash_content(content1);
4090        let hash2 = indexer.hash_content(content2);
4091        let hash3 = indexer.hash_content(content3);
4092
4093        // Same content should produce same hash
4094        assert_eq!(hash1, hash2);
4095
4096        // Different content should produce different hash
4097        assert_ne!(hash1, hash3);
4098
4099        // Hash should be hex string
4100        assert_eq!(hash1.len(), 64); // blake3 hash is 32 bytes = 64 hex chars
4101    }
4102
4103    #[test]
4104    fn test_should_index_rust_file() {
4105        let temp = TempDir::new().unwrap();
4106        let cache = CacheManager::new(temp.path());
4107        let config = IndexConfig::default();
4108        let indexer = Indexer::new(cache, config);
4109
4110        // Create a small Rust file
4111        let rust_file = temp.path().join("test.rs");
4112        fs::write(&rust_file, "fn main() {}").unwrap();
4113
4114        assert!(indexer.should_index(&rust_file));
4115    }
4116
4117    #[test]
4118    fn test_should_index_unsupported_extension() {
4119        let temp = TempDir::new().unwrap();
4120        let cache = CacheManager::new(temp.path());
4121        let config = IndexConfig::default();
4122        let indexer = Indexer::new(cache, config);
4123
4124        // Tracked mode (the default) takes every file by path; whether a file is
4125        // binary is decided from its bytes in `discover_files`, not here.
4126        let unsupported_file = temp.path().join("test.xyz");
4127        fs::write(&unsupported_file, "mystery format").unwrap();
4128        assert!(indexer.should_index(&unsupported_file));
4129
4130        // Allowlist mode keeps the pre-2.0.0 rule.
4131        let allowlist = Indexer::new(
4132            CacheManager::new(temp.path()),
4133            IndexConfig {
4134                mode: IndexMode::Allowlist,
4135                ..Default::default()
4136            },
4137        );
4138        assert!(!allowlist.should_index(&unsupported_file));
4139        let binary_file = temp.path().join("logo.png");
4140        fs::write(&binary_file, "not really a png").unwrap();
4141        assert!(!allowlist.should_index(&binary_file));
4142    }
4143
4144    #[test]
4145    fn test_binary_sniff() {
4146        assert!(!is_binary(b"plain text\n"));
4147        assert!(is_binary(b"\x89PNG\r\n\x1a\n\0\0"));
4148        // Anywhere, not just the first 8 KB: ripgrep skips such a file too.
4149        let mut late = vec![b'a'; 64 * 1024];
4150        late.push(0);
4151        assert!(is_binary(&late));
4152    }
4153
4154    #[test]
4155    fn test_should_index_text_tier() {
4156        let temp = TempDir::new().unwrap();
4157        let indexer = Indexer::new(CacheManager::new(temp.path()), IndexConfig::default());
4158
4159        for name in [
4160            "notes.md",
4161            "config.yaml",
4162            "data.json",
4163            "api.proto",
4164            "run.sh",
4165        ] {
4166            let path = temp.path().join(name);
4167            fs::write(&path, "realm_marker").unwrap();
4168            assert!(indexer.should_index(&path), "{name} should be indexed");
4169        }
4170
4171        // Lock files are indexed in tracked mode (and left out of searches unless
4172        // asked for); allowlist mode never indexes them.
4173        let lock = temp.path().join("package-lock.json");
4174        fs::write(&lock, "{}").unwrap();
4175        assert!(indexer.should_index(&lock));
4176        let allowlist = Indexer::new(
4177            CacheManager::new(temp.path()),
4178            IndexConfig {
4179                mode: IndexMode::Allowlist,
4180                ..Default::default()
4181            },
4182        );
4183        assert!(!allowlist.should_index(&lock));
4184    }
4185
4186    #[test]
4187    fn test_text_tier_can_be_disabled() {
4188        let temp = TempDir::new().unwrap();
4189        let indexer = Indexer::new(
4190            CacheManager::new(temp.path()),
4191            IndexConfig {
4192                text_tier: false,
4193                ..Default::default()
4194            },
4195        );
4196
4197        let md = temp.path().join("notes.md");
4198        fs::write(&md, "text").unwrap();
4199        assert!(!indexer.should_index(&md));
4200
4201        let rs = temp.path().join("main.rs");
4202        fs::write(&rs, "fn main() {}").unwrap();
4203        assert!(indexer.should_index(&rs), "code is unaffected");
4204    }
4205
4206    #[test]
4207    fn test_should_index_no_extension() {
4208        let temp = TempDir::new().unwrap();
4209        let cache = CacheManager::new(temp.path());
4210        let config = IndexConfig::default();
4211        let indexer = Indexer::new(cache, config);
4212
4213        // Tracked mode: every extensionless text file. Allowlist mode: only the
4214        // names on the list (`Makefile`), not `README`.
4215        let makefile = temp.path().join("Makefile");
4216        fs::write(&makefile, "all:\n\techo hello").unwrap();
4217        assert!(indexer.should_index(&makefile));
4218
4219        let readme = temp.path().join("README");
4220        fs::write(&readme, "hello").unwrap();
4221        assert!(indexer.should_index(&readme));
4222
4223        let allowlist = Indexer::new(
4224            CacheManager::new(temp.path()),
4225            IndexConfig {
4226                mode: IndexMode::Allowlist,
4227                ..Default::default()
4228            },
4229        );
4230        assert!(allowlist.should_index(&makefile));
4231        assert!(!allowlist.should_index(&readme));
4232    }
4233
4234    #[test]
4235    fn test_should_index_size_limit() {
4236        let temp = TempDir::new().unwrap();
4237        let cache = CacheManager::new(temp.path());
4238
4239        // Config with 100 byte size limit
4240        let config = IndexConfig {
4241            max_file_size: 100,
4242            ..Default::default()
4243        };
4244
4245        let indexer = Indexer::new(cache, config);
4246
4247        // Create small file (should be indexed)
4248        let small_file = temp.path().join("small.rs");
4249        fs::write(&small_file, "fn main() {}").unwrap();
4250        assert!(indexer.should_index(&small_file));
4251
4252        // Create large file (should be skipped)
4253        let large_file = temp.path().join("large.rs");
4254        let large_content = "a".repeat(150);
4255        fs::write(&large_file, large_content).unwrap();
4256        assert!(!indexer.should_index(&large_file));
4257    }
4258
4259    #[test]
4260    fn test_discover_files_empty_dir() {
4261        let temp = TempDir::new().unwrap();
4262        let cache = CacheManager::new(temp.path());
4263        let config = IndexConfig::default();
4264        let indexer = Indexer::new(cache, config);
4265
4266        let files = indexer
4267            .discover_files(temp.path(), &HashMap::new())
4268            .unwrap()
4269            .rels
4270            .iter()
4271            .map(std::path::PathBuf::from)
4272            .collect::<Vec<_>>();
4273        assert_eq!(files.len(), 0);
4274    }
4275
4276    #[test]
4277    fn test_discover_files_single_file() {
4278        let temp = TempDir::new().unwrap();
4279        let cache = CacheManager::new(temp.path());
4280        let config = IndexConfig::default();
4281        let indexer = Indexer::new(cache, config);
4282
4283        // Create a Rust file
4284        let rust_file = temp.path().join("main.rs");
4285        fs::write(&rust_file, "fn main() {}").unwrap();
4286
4287        let files = indexer
4288            .discover_files(temp.path(), &HashMap::new())
4289            .unwrap()
4290            .rels
4291            .iter()
4292            .map(std::path::PathBuf::from)
4293            .collect::<Vec<_>>();
4294        assert_eq!(files.len(), 1);
4295        assert!(files[0].ends_with("main.rs"));
4296    }
4297
4298    #[test]
4299    fn test_discover_files_multiple_languages() {
4300        let temp = TempDir::new().unwrap();
4301        let cache = CacheManager::new(temp.path());
4302        let config = IndexConfig::default();
4303        let indexer = Indexer::new(cache, config);
4304
4305        // Create files of different languages
4306        fs::write(temp.path().join("main.rs"), "fn main() {}").unwrap();
4307        fs::write(temp.path().join("script.py"), "print('hello')").unwrap();
4308        fs::write(temp.path().join("app.js"), "console.log('hi')").unwrap();
4309        // Since 1.7.2 markdown IS indexed, in the plain-text tier.
4310        fs::write(temp.path().join("README.md"), "# Project").unwrap();
4311        // Since 2.0.0 (tracked mode) every non-binary file is indexed, whatever
4312        // its extension; a binary one is sniffed out.
4313        fs::write(temp.path().join("mystery.xyz"), "?").unwrap();
4314        fs::write(temp.path().join("blob.bin"), b"\0\x01\x02").unwrap();
4315
4316        let found = indexer
4317            .discover_files(temp.path(), &HashMap::new())
4318            .unwrap();
4319        assert_eq!(found.rels.len(), 5, "3 code files, the markdown, the .xyz");
4320        assert_eq!(found.skipped_binary, 1);
4321    }
4322
4323    #[test]
4324    fn test_discover_files_subdirectories() {
4325        let temp = TempDir::new().unwrap();
4326        let cache = CacheManager::new(temp.path());
4327        let config = IndexConfig::default();
4328        let indexer = Indexer::new(cache, config);
4329
4330        // Create nested directory structure
4331        let src_dir = temp.path().join("src");
4332        fs::create_dir(&src_dir).unwrap();
4333        fs::write(src_dir.join("main.rs"), "fn main() {}").unwrap();
4334        fs::write(src_dir.join("lib.rs"), "pub mod test {}").unwrap();
4335
4336        let tests_dir = temp.path().join("tests");
4337        fs::create_dir(&tests_dir).unwrap();
4338        fs::write(tests_dir.join("test.rs"), "#[test] fn test() {}").unwrap();
4339
4340        let files = indexer
4341            .discover_files(temp.path(), &HashMap::new())
4342            .unwrap()
4343            .rels
4344            .iter()
4345            .map(std::path::PathBuf::from)
4346            .collect::<Vec<_>>();
4347        assert_eq!(files.len(), 3);
4348    }
4349
4350    #[test]
4351    fn test_discover_files_respects_gitignore() {
4352        let temp = TempDir::new().unwrap();
4353
4354        // Initialize git repo (required for .gitignore to work with WalkBuilder)
4355        std::process::Command::new("git")
4356            .arg("init")
4357            .current_dir(temp.path())
4358            .output()
4359            .expect("Failed to initialize git repo");
4360
4361        let cache = CacheManager::new(temp.path());
4362        let config = IndexConfig::default();
4363        let indexer = Indexer::new(cache, config);
4364
4365        // Create .gitignore - use "ignored/" pattern to ignore the directory
4366        // Note: WalkBuilder respects .gitignore ONLY in git repositories
4367        fs::write(temp.path().join(".gitignore"), "ignored/\n").unwrap();
4368
4369        // Create files
4370        fs::write(temp.path().join("included.rs"), "fn main() {}").unwrap();
4371        fs::write(temp.path().join("also_included.py"), "print('hi')").unwrap();
4372
4373        let ignored_dir = temp.path().join("ignored");
4374        fs::create_dir(&ignored_dir).unwrap();
4375        fs::write(ignored_dir.join("excluded.rs"), "fn test() {}").unwrap();
4376
4377        let files = indexer
4378            .discover_files(temp.path(), &HashMap::new())
4379            .unwrap()
4380            .rels
4381            .iter()
4382            .map(std::path::PathBuf::from)
4383            .collect::<Vec<_>>();
4384
4385        // Verify the expected files are found
4386        assert!(
4387            files.iter().any(|f| f.ends_with("included.rs")),
4388            "Should find included.rs"
4389        );
4390        assert!(
4391            files.iter().any(|f| f.ends_with("also_included.py")),
4392            "Should find also_included.py"
4393        );
4394
4395        // Verify excluded.rs in ignored/ directory is NOT found
4396        // This is the key test - gitignore should filter it out
4397        assert!(
4398            !files.iter().any(|f| {
4399                let path_str = f.to_string_lossy();
4400                path_str.contains("ignored") && f.ends_with("excluded.rs")
4401            }),
4402            "Should NOT find excluded.rs in ignored/ directory (gitignore pattern)"
4403        );
4404
4405        // Should find exactly 2 files (included.rs and also_included.py)
4406        // .gitignore file itself has no supported language extension, so it won't be indexed
4407        assert_eq!(
4408            files.len(),
4409            2,
4410            "Should find exactly 2 files (not including .gitignore or ignored/excluded.rs)"
4411        );
4412    }
4413
4414    #[test]
4415    fn test_index_empty_directory() {
4416        let temp = TempDir::new().unwrap();
4417        let cache = CacheManager::new(temp.path());
4418        let config = IndexConfig::default();
4419        let indexer = Indexer::new(cache, config);
4420
4421        let stats = indexer.index(temp.path(), false).unwrap();
4422
4423        assert_eq!(stats.total_files, 0);
4424    }
4425
4426    #[test]
4427    fn test_index_single_rust_file() {
4428        let temp = TempDir::new().unwrap();
4429        let project_root = temp.path().join("project");
4430        fs::create_dir(&project_root).unwrap();
4431
4432        let cache = CacheManager::new(&project_root);
4433        let config = IndexConfig::default();
4434        let indexer = Indexer::new(cache, config);
4435
4436        // Create a Rust file
4437        fs::write(
4438            project_root.join("main.rs"),
4439            "fn main() { println!(\"Hello\"); }",
4440        )
4441        .unwrap();
4442
4443        let stats = indexer.index(&project_root, false).unwrap();
4444
4445        assert_eq!(stats.total_files, 1);
4446        assert!(stats.files_by_language.contains_key("Rust"));
4447    }
4448
4449    #[test]
4450    fn test_index_multiple_files() {
4451        let temp = TempDir::new().unwrap();
4452        let project_root = temp.path().join("project");
4453        fs::create_dir(&project_root).unwrap();
4454
4455        let cache = CacheManager::new(&project_root);
4456        let config = IndexConfig::default();
4457        let indexer = Indexer::new(cache, config);
4458
4459        // Create multiple files
4460        fs::write(project_root.join("main.rs"), "fn main() {}").unwrap();
4461        fs::write(project_root.join("lib.rs"), "pub fn test() {}").unwrap();
4462        fs::write(project_root.join("script.py"), "def main(): pass").unwrap();
4463
4464        let stats = indexer.index(&project_root, false).unwrap();
4465
4466        assert_eq!(stats.total_files, 3);
4467        assert_eq!(stats.files_by_language.get("Rust"), Some(&2));
4468        assert_eq!(stats.files_by_language.get("Python"), Some(&1));
4469    }
4470
4471    #[test]
4472    fn test_index_creates_trigram_index() {
4473        let temp = TempDir::new().unwrap();
4474        let project_root = temp.path().join("project");
4475        fs::create_dir(&project_root).unwrap();
4476
4477        let cache = CacheManager::new(&project_root);
4478        let config = IndexConfig::default();
4479        let indexer = Indexer::new(cache, config);
4480
4481        fs::write(project_root.join("main.rs"), "fn main() {}").unwrap();
4482
4483        indexer.index(&project_root, false).unwrap();
4484
4485        // Verify trigrams.bin was created
4486        let trigrams_path = project_root.join(".reflex/trigrams.bin");
4487        assert!(trigrams_path.exists());
4488    }
4489
4490    #[test]
4491    fn test_index_creates_content_store() {
4492        let temp = TempDir::new().unwrap();
4493        let project_root = temp.path().join("project");
4494        fs::create_dir(&project_root).unwrap();
4495
4496        let cache = CacheManager::new(&project_root);
4497        let config = IndexConfig::default();
4498        let indexer = Indexer::new(cache, config);
4499
4500        fs::write(project_root.join("main.rs"), "fn main() {}").unwrap();
4501
4502        indexer.index(&project_root, false).unwrap();
4503
4504        // Verify content.bin was created
4505        let content_path = project_root.join(".reflex/content.bin");
4506        assert!(content_path.exists());
4507    }
4508
4509    #[test]
4510    fn test_index_incremental_no_changes() {
4511        let temp = TempDir::new().unwrap();
4512        let project_root = temp.path().join("project");
4513        fs::create_dir(&project_root).unwrap();
4514
4515        let cache = CacheManager::new(&project_root);
4516        let config = IndexConfig::default();
4517        let indexer = Indexer::new(cache, config);
4518
4519        fs::write(project_root.join("main.rs"), "fn main() {}").unwrap();
4520
4521        // First index
4522        let stats1 = indexer.index(&project_root, false).unwrap();
4523        assert_eq!(stats1.total_files, 1);
4524
4525        // Second index without changes
4526        let stats2 = indexer.index(&project_root, false).unwrap();
4527        assert_eq!(stats2.total_files, 1);
4528    }
4529
4530    #[test]
4531    fn test_index_incremental_with_changes() {
4532        let temp = TempDir::new().unwrap();
4533        let project_root = temp.path().join("project");
4534        fs::create_dir(&project_root).unwrap();
4535
4536        let cache = CacheManager::new(&project_root);
4537        let config = IndexConfig::default();
4538        let indexer = Indexer::new(cache, config);
4539
4540        let main_path = project_root.join("main.rs");
4541        fs::write(&main_path, "fn main() {}").unwrap();
4542
4543        // First index
4544        indexer.index(&project_root, false).unwrap();
4545
4546        // Modify file
4547        fs::write(&main_path, "fn main() { println!(\"changed\"); }").unwrap();
4548
4549        // Second index should detect change
4550        let stats = indexer.index(&project_root, false).unwrap();
4551        assert_eq!(stats.total_files, 1);
4552    }
4553
4554    #[test]
4555    fn test_index_incremental_new_file() {
4556        let temp = TempDir::new().unwrap();
4557        let project_root = temp.path().join("project");
4558        fs::create_dir(&project_root).unwrap();
4559
4560        let cache = CacheManager::new(&project_root);
4561        let config = IndexConfig::default();
4562        let indexer = Indexer::new(cache, config);
4563
4564        fs::write(project_root.join("main.rs"), "fn main() {}").unwrap();
4565
4566        // First index
4567        let stats1 = indexer.index(&project_root, false).unwrap();
4568        assert_eq!(stats1.total_files, 1);
4569
4570        // Add new file
4571        fs::write(project_root.join("lib.rs"), "pub fn test() {}").unwrap();
4572
4573        // Second index should include new file
4574        let stats2 = indexer.index(&project_root, false).unwrap();
4575        assert_eq!(stats2.total_files, 2);
4576    }
4577
4578    #[test]
4579    fn test_index_parallel_threads_config() {
4580        let temp = TempDir::new().unwrap();
4581        let project_root = temp.path().join("project");
4582        fs::create_dir(&project_root).unwrap();
4583
4584        let cache = CacheManager::new(&project_root);
4585
4586        // Test with explicit thread count
4587        let config = IndexConfig {
4588            parallel_threads: 2,
4589            ..Default::default()
4590        };
4591
4592        let indexer = Indexer::new(cache, config);
4593
4594        fs::write(project_root.join("main.rs"), "fn main() {}").unwrap();
4595
4596        let stats = indexer.index(&project_root, false).unwrap();
4597        assert_eq!(stats.total_files, 1);
4598    }
4599
4600    #[test]
4601    fn test_index_parallel_threads_auto() {
4602        let temp = TempDir::new().unwrap();
4603        let project_root = temp.path().join("project");
4604        fs::create_dir(&project_root).unwrap();
4605
4606        let cache = CacheManager::new(&project_root);
4607
4608        // Test with auto thread count (0 = auto)
4609        let config = IndexConfig {
4610            parallel_threads: 0,
4611            ..Default::default()
4612        };
4613
4614        let indexer = Indexer::new(cache, config);
4615
4616        fs::write(project_root.join("main.rs"), "fn main() {}").unwrap();
4617
4618        let stats = indexer.index(&project_root, false).unwrap();
4619        assert_eq!(stats.total_files, 1);
4620    }
4621
4622    #[test]
4623    fn test_index_respects_size_limit() {
4624        let temp = TempDir::new().unwrap();
4625        let project_root = temp.path().join("project");
4626        fs::create_dir(&project_root).unwrap();
4627
4628        let cache = CacheManager::new(&project_root);
4629
4630        // Very small size limit
4631        let config = IndexConfig {
4632            max_file_size: 50,
4633            ..Default::default()
4634        };
4635
4636        let indexer = Indexer::new(cache, config);
4637
4638        // Small file (should be indexed)
4639        fs::write(project_root.join("small.rs"), "fn a() {}").unwrap();
4640
4641        // Large file (should be skipped)
4642        let large_content = "fn main() {}\n".repeat(10);
4643        fs::write(project_root.join("large.rs"), large_content).unwrap();
4644
4645        let stats = indexer.index(&project_root, false).unwrap();
4646
4647        // Only small file should be indexed
4648        assert_eq!(stats.total_files, 1);
4649    }
4650
4651    #[test]
4652    fn test_index_mixed_languages() {
4653        let temp = TempDir::new().unwrap();
4654        let project_root = temp.path().join("project");
4655        fs::create_dir(&project_root).unwrap();
4656
4657        let cache = CacheManager::new(&project_root);
4658        let config = IndexConfig::default();
4659        let indexer = Indexer::new(cache, config);
4660
4661        // Create files in multiple languages
4662        fs::write(project_root.join("main.rs"), "fn main() {}").unwrap();
4663        fs::write(project_root.join("test.py"), "def test(): pass").unwrap();
4664        fs::write(project_root.join("app.js"), "function main() {}").unwrap();
4665        fs::write(project_root.join("lib.go"), "func main() {}").unwrap();
4666
4667        let stats = indexer.index(&project_root, false).unwrap();
4668
4669        assert_eq!(stats.total_files, 4);
4670        assert!(stats.files_by_language.contains_key("Rust"));
4671        assert!(stats.files_by_language.contains_key("Python"));
4672        assert!(stats.files_by_language.contains_key("JavaScript"));
4673        assert!(stats.files_by_language.contains_key("Go"));
4674    }
4675
4676    #[test]
4677    fn test_index_updates_cache_stats() {
4678        let temp = TempDir::new().unwrap();
4679        let project_root = temp.path().join("project");
4680        fs::create_dir(&project_root).unwrap();
4681
4682        let cache = CacheManager::new(&project_root);
4683        let config = IndexConfig::default();
4684        let indexer = Indexer::new(cache, config);
4685
4686        fs::write(project_root.join("main.rs"), "fn main() {}").unwrap();
4687
4688        indexer.index(&project_root, false).unwrap();
4689
4690        // Verify cache stats were updated
4691        let cache = CacheManager::new(&project_root);
4692        let stats = cache.stats().unwrap();
4693
4694        assert_eq!(stats.total_files, 1);
4695        assert!(stats.index_size_bytes > 0);
4696    }
4697}