1use anyhow::{Context, Result};
7use ignore::WalkBuilder;
8use indicatif::{ProgressBar, ProgressDrawTarget, ProgressStyle};
9use rayon::prelude::*;
10use std::collections::HashMap;
11use std::path::{Path, PathBuf};
12use std::sync::atomic::{AtomicU64, Ordering};
13use std::sync::{Arc, Mutex};
14use std::time::Instant;
15
16use crate::cache::{CacheManager, WALK_SEQ_GAP};
17use crate::content_store::ContentWriter;
18use crate::models::{IndexConfig, IndexMode, IndexStats, Language};
19#[cfg(unix)]
20use crate::output;
21use crate::parsers::c::CDependencyExtractor;
22use crate::parsers::cpp::CppDependencyExtractor;
23use crate::parsers::csharp::CSharpDependencyExtractor;
24use crate::parsers::go::GoDependencyExtractor;
25use crate::parsers::java::JavaDependencyExtractor;
26use crate::parsers::kotlin::KotlinDependencyExtractor;
27use crate::parsers::php::PhpDependencyExtractor;
28use crate::parsers::python::PythonDependencyExtractor;
29use crate::parsers::ruby::RubyDependencyExtractor;
30use crate::parsers::rust::RustDependencyExtractor;
31use crate::parsers::svelte::SvelteDependencyExtractor;
32use crate::parsers::swift_imports::SwiftDependencyExtractor;
33use crate::parsers::typescript::TypeScriptDependencyExtractor;
34use crate::parsers::vue::VueDependencyExtractor;
35use crate::parsers::zig::ZigDependencyExtractor;
36use crate::parsers::{DependencyExtractor, ExportInfo, ImportInfo};
37use crate::trigram_build::{TrigramIndexBuilder, TrigramRun};
38
39pub type ProgressCallback = Arc<dyn Fn(usize, usize, String) + Send + Sync>;
42
43struct FileProcessingResult {
45 hash: String,
46 content: String,
47 language: Language,
48 line_count: usize,
49 size: u64,
52 mtime_ns: i64,
53 imports: Option<Extracted>,
56 trigram_run: TrigramRun,
58}
59
60const RESOLVER_DIGEST_KEY: &str = "resolver_config_digest";
63
64pub const INDEX_GENERATION_KEY: &str = "index_generation";
67
68const RUN_MARKER: &str = ".index-run";
70
71#[derive(Debug, Clone, Copy, PartialEq, Eq)]
73enum FileStatus {
74 Unchanged,
76 Touched,
78 Modified,
80 Added,
82}
83
84struct IndexedFile {
86 index: usize,
88 hash: String,
89 language: Language,
90 line_count: usize,
91 size: u64,
92 mtime_ns: i64,
93 imports: Option<Extracted>,
96}
97
98struct Rewrite {
100 rel: String,
101 size: u64,
103 walk_seq: i64,
104}
105
106struct DirtyFlags {
110 rewrites: std::collections::HashSet<String>,
111 touched: Vec<(i64, u64, i64, bool)>,
112 flips: Vec<(i64, bool)>,
113 git_dirty: bool,
114}
115
116struct DeltaChanges<'a> {
120 root: &'a Path,
121 rewrites: Vec<Rewrite>,
123 deleted: Vec<(String, i64)>,
125 walk_moves: Vec<(i64, i64)>,
127 dirty: Box<dyn FnOnce() -> DirtyFlags + 'a>,
130 stored: HashMap<String, crate::meta_update::StoredFile>,
132 sync_all_branch_rows: bool,
135 full_stats: bool,
138 live_files: usize,
140 breakdown_rest: (usize, usize, usize),
143 branch_hashes: HashMap<String, String>,
145 branch: &'a str,
146 commit: Option<&'a str>,
147 resolver_configs: &'a crate::dependency_resolve::ResolverConfigs,
148 run_start: std::time::SystemTime,
149 generation: u64,
150 pool: &'a rayon::ThreadPool,
151 deleted_file_count: usize,
152 skipped: (usize, u64, usize),
154}
155
156struct ProcessCtx<'a> {
158 root: &'a Path,
159 rels: &'a [String],
160 stored: &'a HashMap<String, crate::meta_update::StoredFile>,
161 run_start: std::time::SystemTime,
162 full_deps: bool,
163 tsconfigs: &'a HashMap<PathBuf, crate::parsers::tsconfig::PathAliasMap>,
164}
165
166struct RefreshUnchanged<'a> {
168 rels: &'a [String],
169 metas: &'a [Option<crate::cache::FileStat>],
170 status: &'a [FileStatus],
171 rows: &'a [Option<&'a crate::meta_update::StoredFile>],
173 existing_hashes: &'a BranchHashes<'a>,
174 dirty_paths: &'a std::collections::HashSet<String>,
175 branch: &'a str,
176 in_sync: bool,
178 commit: Option<&'a str>,
179 git_dirty: Option<bool>,
181 run_start: std::time::SystemTime,
182 skipped: (usize, u64, usize),
184}
185
186struct RefreshNamed<'a> {
188 touched: &'a [(i64, u64, i64, bool)],
189 flips: &'a [(i64, bool)],
190 walk_moves: &'a [(i64, i64)],
191 branch: &'a str,
192 commit: Option<&'a str>,
193 git_dirty: bool,
194 live_files: usize,
195 skipped: (usize, u64, usize),
197}
198
199struct MetaWrite<'a> {
201 root: &'a Path,
202 rels: &'a [String],
203 present: &'a [usize],
205 written: &'a [IndexedFile],
207 metas: &'a [Option<crate::cache::FileStat>],
208 run_start: std::time::SystemTime,
209 stored: &'a HashMap<String, crate::meta_update::StoredFile>,
210 status: &'a [FileStatus],
211 dirty_paths: &'a std::collections::HashSet<String>,
212 branch: &'a str,
213 commit: Option<&'a str>,
214 resolver_configs: &'a crate::dependency_resolve::ResolverConfigs,
215 full_deps: bool,
216 generation: u64,
218}
219
220enum BranchHashes<'a> {
225 Synced(&'a HashMap<String, crate::meta_update::StoredFile>),
226 Loaded(HashMap<String, String>),
227}
228
229impl BranchHashes<'_> {
230 fn get(&self, path: &str) -> Option<&str> {
231 match self {
232 Self::Synced(stored) => stored.get(path).map(|row| row.hash.as_str()),
233 Self::Loaded(hashes) => hashes.get(path).map(String::as_str),
234 }
235 }
236}
237
238fn breakdown<'a>(
239 files: impl Iterator<Item = (&'a str, &'a str)>,
240 existing_hashes: &BranchHashes<'_>,
241) -> (usize, usize, usize) {
242 let (mut new, mut modified, mut unchanged) = (0, 0, 0);
243 for (path, hash) in files {
244 match existing_hashes.get(path) {
245 None => new += 1,
246 Some(old) if old != hash => modified += 1,
247 _ => unchanged += 1,
248 }
249 }
250 (new, modified, unchanged)
251}
252
253fn refresh_vendored(
256 tx: &rusqlite::Connection,
257 rules: &crate::vendor::VendorRules,
258 ids: Option<&[i64]>,
259) -> Result<usize> {
260 let mut changed: Vec<(i64, bool)> = Vec::new();
261 {
262 let mut visit = |id: i64, path: &str, old: bool| {
263 let new = rules.is_vendored(path, crate::models::Language::from_path(Path::new(path)));
264 if new != old {
265 changed.push((id, new));
266 }
267 };
268 match ids {
269 None => {
270 let mut stmt = tx.prepare("SELECT id, path, vendored FROM files")?;
271 let rows = stmt.query_map([], |r| {
272 Ok((
273 r.get::<_, i64>(0)?,
274 r.get::<_, String>(1)?,
275 r.get::<_, bool>(2)?,
276 ))
277 })?;
278 for row in rows {
279 let (id, path, old) = row?;
280 visit(id, &path, old);
281 }
282 }
283 Some(ids) => {
284 let mut stmt = tx.prepare("SELECT path, vendored FROM files WHERE id = ?")?;
285 for &id in ids {
286 let (path, old): (String, bool) =
287 stmt.query_row([id], |r| Ok((r.get(0)?, r.get(1)?)))?;
288 visit(id, &path, old);
289 }
290 }
291 }
292 }
293 let mut stmt = tx.prepare("UPDATE files SET vendored = ? WHERE id = ?")?;
294 for (id, vendored) in &changed {
295 stmt.execute(rusqlite::params![vendored, id])?;
296 }
297 Ok(changed.len())
298}
299
300fn reresolve(
306 tx: &rusqlite::Connection,
307 ctx: &crate::dependency_resolve::ResolverContext<'_>,
308 resolver: &crate::dependency::PathResolver,
309 skip: &std::collections::HashSet<i64>,
310) -> Result<usize> {
311 let mut changed: Vec<(i64, Option<i64>)> = Vec::new();
312 {
313 let mut stmt = tx.prepare(
314 "SELECT d.id, d.file_id, f.path, d.imported_path, d.resolved_file_id
315 FROM file_dependencies d JOIN files f ON f.id = d.file_id
316 WHERE d.import_type NOT IN ('external', 'stdlib')
317 AND d.resolved_package IS NULL",
318 )?;
319 let rows = stmt.query_map([], |r| {
320 Ok((
321 r.get::<_, i64>(0)?,
322 r.get::<_, i64>(1)?,
323 r.get::<_, String>(2)?,
324 r.get::<_, String>(3)?,
325 r.get::<_, Option<i64>>(4)?,
326 ))
327 })?;
328 for row in rows {
329 let (id, file_id, path, imported_path, old) = row?;
330 if skip.contains(&file_id) {
331 continue;
332 }
333 let import = ImportInfo {
336 imported_path,
337 import_type: crate::models::ImportType::Internal,
338 line_number: 0,
339 imported_symbols: None,
340 };
341 let new = ctx.resolve_import(&path, &import, resolver);
342 if new != old {
343 changed.push((id, new));
344 }
345 }
346 }
347 let mut stmt = tx.prepare("UPDATE file_dependencies SET resolved_file_id = ? WHERE id = ?")?;
348 for (id, new) in &changed {
349 stmt.execute(rusqlite::params![new, id])?;
350 }
351 let mut total = changed.len();
352
353 let mut changed: Vec<(i64, Option<i64>)> = Vec::new();
354 {
355 let mut stmt = tx.prepare(
356 "SELECT e.id, e.file_id, f.path, e.source_path, e.resolved_source_id
357 FROM file_exports e JOIN files f ON f.id = e.file_id",
358 )?;
359 let rows = stmt.query_map([], |r| {
360 Ok((
361 r.get::<_, i64>(0)?,
362 r.get::<_, i64>(1)?,
363 r.get::<_, String>(2)?,
364 r.get::<_, String>(3)?,
365 r.get::<_, Option<i64>>(4)?,
366 ))
367 })?;
368 for row in rows {
369 let (id, file_id, path, source_path, old) = row?;
370 if skip.contains(&file_id) {
371 continue;
372 }
373 let export = ExportInfo {
374 exported_symbol: None,
375 source_path,
376 line_number: 0,
377 };
378 let new = ctx.resolve_export(&path, &export, resolver);
379 if new != old {
380 changed.push((id, new));
381 }
382 }
383 }
384 let mut stmt = tx.prepare("UPDATE file_exports SET resolved_source_id = ? WHERE id = ?")?;
385 for (id, new) in &changed {
386 stmt.execute(rusqlite::params![new, id])?;
387 }
388 total += changed.len();
389 Ok(total)
390}
391
392type Extracted = (Vec<ImportInfo>, Vec<ExportInfo>, Vec<(String, String)>);
395
396fn extract_imports(
399 language: Language,
400 content: &str,
401 path_str: &str,
402 root: &Path,
403 tsconfigs: &HashMap<PathBuf, crate::parsers::tsconfig::PathAliasMap>,
404) -> Extracted {
405 let mut parsed_exports: Vec<ExportInfo> = Vec::new();
407 let mut declared_namespaces: Vec<(String, String)> = Vec::new();
408 let dependencies = match language {
409 Language::Rust => match RustDependencyExtractor::extract_dependencies(content) {
410 Ok(deps) => deps,
411 Err(e) => {
412 log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
413 Vec::new()
414 }
415 },
416 Language::Python => match PythonDependencyExtractor::extract_dependencies(content) {
417 Ok(deps) => deps,
418 Err(e) => {
419 log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
420 Vec::new()
421 }
422 },
423 Language::TypeScript if crate::parsers::is_qt_linguist_ts(content) => Vec::new(),
425 Language::TypeScript | Language::JavaScript => {
426 let alias_map =
429 crate::dependency_resolve::find_nearest_tsconfig(path_str, root, tsconfigs);
430 match TypeScriptDependencyExtractor::extract_dependencies_and_exports(
431 content, alias_map,
432 ) {
433 Ok((deps, exports)) => {
434 parsed_exports = exports;
435 deps
436 }
437 Err(e) => {
438 log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
439 Vec::new()
440 }
441 }
442 }
443 Language::Go => match GoDependencyExtractor::extract_dependencies(content) {
444 Ok(deps) => deps,
445 Err(e) => {
446 log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
447 Vec::new()
448 }
449 },
450 Language::Java => match JavaDependencyExtractor::extract_dependencies(content) {
451 Ok(deps) => deps,
452 Err(e) => {
453 log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
454 Vec::new()
455 }
456 },
457 Language::C => match CDependencyExtractor::extract_dependencies(content) {
458 Ok(deps) => deps,
459 Err(e) => {
460 log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
461 Vec::new()
462 }
463 },
464 Language::Cpp => match CppDependencyExtractor::extract_dependencies(content) {
465 Ok(deps) => deps,
466 Err(e) => {
467 log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
468 Vec::new()
469 }
470 },
471 Language::CSharp => {
472 match CSharpDependencyExtractor::extract_dependencies_and_members(content) {
473 Ok((deps, members)) => {
474 declared_namespaces = members;
475 deps
476 }
477 Err(e) => {
478 log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
479 Vec::new()
480 }
481 }
482 }
483 Language::PHP => match PhpDependencyExtractor::extract_dependencies(content) {
484 Ok(deps) => deps,
485 Err(e) => {
486 log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
487 Vec::new()
488 }
489 },
490 Language::Ruby => match RubyDependencyExtractor::extract_dependencies(content) {
491 Ok(deps) => deps,
492 Err(e) => {
493 log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
494 Vec::new()
495 }
496 },
497 Language::Kotlin => match KotlinDependencyExtractor::extract_dependencies(content) {
498 Ok(deps) => deps,
499 Err(e) => {
500 log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
501 Vec::new()
502 }
503 },
504 Language::Zig => match ZigDependencyExtractor::extract_dependencies(content) {
505 Ok(deps) => deps,
506 Err(e) => {
507 log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
508 Vec::new()
509 }
510 },
511 Language::Swift => match SwiftDependencyExtractor::extract_dependencies(content) {
513 Ok(deps) => deps,
514 Err(e) => {
515 log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
516 Vec::new()
517 }
518 },
519 Language::Vue => {
520 let alias_map =
523 crate::dependency_resolve::find_nearest_tsconfig(path_str, root, tsconfigs);
524 match VueDependencyExtractor::extract_dependencies_and_exports(content, alias_map) {
525 Ok((deps, exports)) => {
526 parsed_exports = exports;
527 deps
528 }
529 Err(e) => {
530 log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
531 Vec::new()
532 }
533 }
534 }
535 Language::Svelte => match SvelteDependencyExtractor::extract_dependencies(content) {
536 Ok(deps) => deps,
537 Err(e) => {
538 log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
539 Vec::new()
540 }
541 },
542 _ => Vec::new(),
544 };
545
546 let members = match language {
549 Language::Java | Language::Kotlin => {
550 crate::parsers::java::jvm_package_members(path_str, content)
551 }
552 Language::CSharp => declared_namespaces,
553 _ => Vec::new(),
554 };
555 (dependencies, parsed_exports, members)
556}
557
558pub struct Indexer {
560 cache: CacheManager,
561 config: IndexConfig,
562 batch_limits: Option<(usize, u64)>,
564 merge_limits: Option<(usize, u64)>,
567 recent_limits: Option<(usize, u64)>,
571 abort_at: Option<String>,
573 rules_pending: std::sync::Mutex<Option<String>>,
576}
577
578static RESOLVER_CACHE: Mutex<Option<(PathBuf, u64, crate::dependency::PathResolver)>> =
584 Mutex::new(None);
585
586pub const RECENT_MAX_FILES: usize = 256;
588pub const RECENT_BYTES_DIVISOR: u64 = 16;
590
591#[doc(hidden)]
593pub const ABORT_EXIT_CODE: i32 = 86;
594
595pub const DELTA_MAX_FILES: usize = 2000;
597pub const DELTA_MAX_CORPUS_PERCENT: u64 = 5;
600
601pub const BATCH_MAX_FILES: usize = 5000;
603pub const BATCH_MAX_BYTES: u64 = 48 << 20;
608
609pub fn plan_batches(
612 sizes: &[u64],
613 max_files: usize,
614 max_bytes: u64,
615) -> Vec<std::ops::Range<usize>> {
616 let max_files = max_files.max(1);
617 let mut batches = Vec::new();
618 let mut start = 0usize;
619 let mut bytes = 0u64;
620 for (i, &size) in sizes.iter().enumerate() {
621 let count = i - start;
622 if count > 0 && (count >= max_files || bytes.saturating_add(size) > max_bytes) {
623 batches.push(start..i);
624 start = i;
625 bytes = 0;
626 }
627 bytes = bytes.saturating_add(size);
628 }
629 if start < sizes.len() {
630 batches.push(start..sizes.len());
631 }
632 batches
633}
634
635#[derive(Clone, Debug)]
648pub struct PathPolicy {
649 overrides: Option<ignore::overrides::Override>,
650 text_tier: bool,
652 languages: Vec<Language>,
654 mode: IndexMode,
656 hidden: bool,
658}
659
660impl Default for PathPolicy {
661 fn default() -> Self {
662 Self {
663 overrides: None,
664 text_tier: true,
665 languages: Vec::new(),
666 mode: IndexMode::Tracked,
667 hidden: false,
668 }
669 }
670}
671
672impl PathPolicy {
673 pub fn from_config(root: &Path, config: &IndexConfig) -> Self {
676 let mut policy = Self {
677 overrides: None,
678 text_tier: config.text_tier,
679 languages: config.languages.clone(),
680 mode: config.mode,
681 hidden: config.hidden,
682 };
683 if config.include_patterns.is_empty() && config.exclude_patterns.is_empty() {
684 return policy;
685 }
686 let mut builder = ignore::overrides::OverrideBuilder::new(root);
687 for pat in &config.include_patterns {
688 if let Err(e) = builder.add(pat) {
689 log::warn!("Invalid [index] include pattern '{}': {}", pat, e);
690 }
691 }
692 for pat in &config.exclude_patterns {
693 let negated = format!("!{}", pat.trim_start_matches('!'));
694 if let Err(e) = builder.add(&negated) {
695 log::warn!("Invalid [index] exclude pattern '{}': {}", pat, e);
696 }
697 }
698 match builder.build() {
699 Ok(ov) => policy.overrides = Some(ov),
700 Err(e) => log::warn!("Failed to build [index] include/exclude policy: {}", e),
701 }
702 policy
703 }
704
705 pub fn classify(&self, path: &Path) -> Option<Language> {
710 if !self.admits(path, false) {
711 return None;
712 }
713 let lang = Language::from_path(path);
714 let name = path.file_name().and_then(|n| n.to_str()).unwrap_or("");
715
716 match lang {
717 Language::Lock | Language::Generated => {
719 (self.mode == IndexMode::Tracked).then_some(lang)
720 }
721 Language::Text => {
728 if !self.text_tier {
729 return None;
730 }
731 match self.mode {
732 IndexMode::Tracked => Some(lang),
733 IndexMode::Allowlist => crate::models::is_text_tier_file(name).then_some(lang),
734 }
735 }
736 Language::Unknown => None,
737 code if !code.is_supported() => {
740 (self.mode == IndexMode::Tracked && self.text_tier).then_some(code)
741 }
742 code => {
743 if !self.languages.is_empty() && !self.languages.contains(&code) {
744 log::debug!(
745 "Skipping {} ({:?} not in configured languages)",
746 path.display(),
747 code
748 );
749 return None;
750 }
751 Some(code)
752 }
753 }
754 }
755
756 pub fn hidden_ok(&self, rel: &Path) -> bool {
763 rel.components()
764 .filter_map(|c| c.as_os_str().to_str())
765 .all(|seg| {
766 if seg == "." || seg == ".." {
767 return true;
768 }
769 if seg == ".git" || seg == crate::cache::CACHE_DIR {
770 return false;
771 }
772 self.hidden || !is_hidden_segment(seg)
773 })
774 }
775
776 pub fn hidden(&self) -> bool {
778 self.hidden
779 }
780
781 pub fn admits(&self, path: &Path, is_dir: bool) -> bool {
784 match &self.overrides {
785 None => true,
786 Some(ov) => !ov.matched(path, is_dir).is_ignore(),
787 }
788 }
789
790 pub fn overrides(&self) -> Option<&ignore::overrides::Override> {
792 self.overrides.as_ref()
793 }
794}
795
796#[derive(Debug, Default)]
798struct Walked {
799 found: Discovered,
800 non_code: Vec<bool>,
802}
803
804#[derive(Debug, Default)]
806struct Discovered {
807 rels: Vec<String>,
810 sizes: Vec<u64>,
812 metas: Vec<Option<crate::cache::FileStat>>,
814 skipped_too_large: usize,
815 skipped_bytes_too_large: u64,
816 skipped_binary: usize,
817}
818
819#[derive(Debug, Default)]
822struct Targets {
823 exact: std::collections::HashSet<String>,
825 ancestors: std::collections::HashSet<String>,
827}
828
829impl Targets {
830 fn new(paths: &[String]) -> Self {
831 let mut out = Self::default();
832 for p in paths {
833 let mut end = p.len();
834 while let Some(slash) = p[..end].rfind('/') {
835 out.ancestors.insert(p[..slash].to_string());
836 end = slash;
837 }
838 out.exact.insert(p.clone());
839 }
840 out
841 }
842
843 fn covers(&self, rel: &str) -> bool {
845 if self.exact.contains(rel) {
846 return true;
847 }
848 let mut end = rel.len();
849 while let Some(slash) = rel[..end].rfind('/') {
850 if self.exact.contains(&rel[..slash]) {
851 return true;
852 }
853 end = slash;
854 }
855 false
856 }
857
858 fn admits(&self, rel: &str) -> bool {
861 rel.is_empty() || self.ancestors.contains(rel) || self.covers(rel)
862 }
863}
864
865fn place_rewrites(
873 conn: &rusqlite::Connection,
874 walk: &mut crate::walk_order::WalkOrder<'_>,
875 rewrites: &[(&str, Option<&crate::meta_update::StoredFile>)],
876 skip: &std::collections::HashSet<i64>,
877) -> Result<Option<HashMap<String, i64>>> {
878 use crate::meta_update::row_near_seq;
879 use std::cmp::Ordering::{Greater, Less};
880 let mut skip = skip.clone();
881 let mut kept: HashMap<&str, i64> = HashMap::new();
882 let mut moved: std::collections::HashSet<&str> = std::collections::HashSet::new();
885 for _ in 0..4 {
886 kept.clear();
887 let mut changed = false;
888 for (path, row) in rewrites {
889 let Some(row) = row else { continue };
890 if moved.contains(path) {
891 continue;
892 }
893 let before = row_near_seq(conn, row.walk_seq, true, &skip)?;
894 let after = row_near_seq(conn, row.walk_seq + 1, false, &skip)?;
895 let fits = before
896 .as_ref()
897 .is_none_or(|(_, p, _)| walk.cmp(p, path) == Some(Less))
898 && after
899 .as_ref()
900 .is_none_or(|(_, p, _)| walk.cmp(path, p) == Some(Less));
901 if fits {
902 kept.insert(path, row.walk_seq);
903 } else {
904 moved.insert(path);
905 skip.insert(row.id);
906 changed = true;
907 }
908 }
909 if !changed {
910 break;
911 }
912 }
913
914 let bounds: (Option<i64>, Option<i64>) =
915 conn.query_row("SELECT MIN(walk_seq), MAX(walk_seq) FROM files", [], |r| {
916 Ok((r.get(0)?, r.get(1)?))
917 })?;
918 let mut gaps: Vec<(&str, Option<i64>, Option<i64>)> = Vec::new();
920 for (path, row) in rewrites {
921 if row.is_some() && !moved.contains(path) {
922 continue;
923 }
924 let (before, after) = match bounds {
925 (Some(min), Some(max)) => {
926 let (mut lo, mut hi) = (min, max + 1);
928 while lo < hi {
929 let mid = lo + (hi - lo) / 2;
930 let after_path = match row_near_seq(conn, mid, false, &skip)? {
931 None => true,
932 Some((_, q, _)) => match walk.cmp(&q, path) {
933 Some(Greater) => true,
934 Some(Less) => false,
935 _ => return Ok(None),
936 },
937 };
938 if after_path {
939 hi = mid;
940 } else {
941 lo = mid + 1;
942 }
943 }
944 (
945 row_near_seq(conn, lo, true, &skip)?.map(|r| r.2),
946 row_near_seq(conn, lo, false, &skip)?.map(|r| r.2),
947 )
948 }
949 _ => (None, None),
950 };
951 gaps.push((path, before, after));
952 }
953 let mut out: HashMap<String, i64> = kept
954 .into_iter()
955 .map(|(p, seq)| (p.to_string(), seq))
956 .collect();
957 let mut i = 0;
959 while i < gaps.len() {
960 let (_, before, after) = gaps[i];
961 let mut j = i;
962 while j < gaps.len() && (gaps[j].1, gaps[j].2) == (before, after) {
963 j += 1;
964 }
965 let k = (j - i) as i64;
966 for (n, (path, _, _)) in gaps[i..j].iter().enumerate() {
967 let n = n as i64;
968 let seq = match (before, after) {
969 (Some(a), Some(b)) => {
970 let step = (b - a) / (k + 1);
971 if step < 1 {
972 return Ok(None);
973 }
974 a.checked_add(step * (n + 1))
975 }
976 (Some(a), None) => a.checked_add(WALK_SEQ_GAP * (n + 1)),
977 (None, Some(b)) => b.checked_sub(WALK_SEQ_GAP * (k - n)),
978 (None, None) => Some(WALK_SEQ_GAP * n),
979 };
980 let Some(seq) = seq else { return Ok(None) };
981 out.insert(path.to_string(), seq);
982 }
983 i = j;
984 }
985 Ok(Some(out))
986}
987
988fn normalize_rel(root: &Path, file_path: &Path) -> String {
992 match file_path.strip_prefix(root) {
993 Ok(rel) => rel.to_string_lossy().replace('\\', "/"),
994 Err(_) => file_path
995 .to_string_lossy()
996 .trim_start_matches("./")
997 .replace('\\', "/"),
998 }
999}
1000
1001pub fn is_hidden_segment(seg: &str) -> bool {
1004 seg.len() > 1 && seg.starts_with('.') && seg != ".."
1005}
1006
1007pub fn is_binary(bytes: &[u8]) -> bool {
1014 memchr::memchr(0, bytes).is_some()
1015}
1016
1017pub fn looks_binary(path: &Path) -> bool {
1021 match std::fs::read(path) {
1022 Ok(bytes) => is_binary(&bytes),
1023 Err(_) => false,
1024 }
1025}
1026
1027struct Laps {
1029 last: Instant,
1030 line: String,
1031}
1032
1033impl Laps {
1034 fn new() -> Self {
1035 Self {
1036 last: Instant::now(),
1037 line: String::new(),
1038 }
1039 }
1040
1041 fn lap(&mut self, name: &str) {
1042 use std::fmt::Write;
1043 let _ = write!(
1044 self.line,
1045 " {}={:.1}",
1046 name,
1047 self.last.elapsed().as_secs_f64() * 1000.0
1048 );
1049 self.last = Instant::now();
1050 }
1051}
1052
1053enum Named {
1055 Skip,
1057 Full,
1060 Path(String),
1062}
1063
1064fn classify_named(root: &Path, abs_root: &Path, path: &Path) -> Named {
1065 let rel = if path.is_absolute() {
1066 match path
1067 .strip_prefix(root)
1068 .or_else(|_| path.strip_prefix(abs_root))
1069 {
1070 Ok(rel) => rel,
1071 Err(_) => return Named::Skip,
1072 }
1073 } else {
1074 path
1075 };
1076 let mut parts: Vec<String> = Vec::new();
1077 for component in rel.components() {
1078 match component {
1079 std::path::Component::Normal(s) => parts.push(s.to_string_lossy().into_owned()),
1080 std::path::Component::CurDir => {}
1081 _ => return Named::Full,
1082 }
1083 }
1084 let Some(name) = parts.last() else {
1085 return Named::Full;
1086 };
1087 if parts[0] == ".git" {
1088 return Named::Skip;
1089 }
1090 if parts[0] == crate::cache::CACHE_DIR {
1091 return if parts.len() == 2 && parts[1] == "config.toml" {
1092 Named::Full
1093 } else {
1094 Named::Skip
1095 };
1096 }
1097 if is_ignore_file_name(name) || crate::dependency_resolve::is_resolver_config_name(name) {
1098 return Named::Full;
1099 }
1100 Named::Path(parts.join("/"))
1101}
1102
1103pub(crate) const RULE_FILES_KEY: &str = "rule_files";
1108
1109pub(crate) const FIXED_RULE_FILES: [&str; 4] =
1112 [".reflex/config.toml", ".gitignore", ".ignore", ".rgignore"];
1113
1114pub(crate) fn is_ignore_file_name(name: &str) -> bool {
1116 matches!(name, ".gitignore" | ".ignore" | ".rgignore")
1117}
1118
1119pub(crate) fn is_rule_file(rel: &str) -> bool {
1121 rel == FIXED_RULE_FILES[0] || is_ignore_file_name(rel.rsplit('/').next().unwrap_or(rel))
1122}
1123
1124pub(crate) fn rule_file_hash(root: &Path, rel: &str) -> String {
1126 match std::fs::read(root.join(rel)) {
1127 Ok(bytes) => blake3::hash(&bytes).to_hex().to_string(),
1128 Err(_) => String::new(),
1129 }
1130}
1131
1132fn rule_files_record<'a>(root: &Path, dirty: impl IntoIterator<Item = &'a String>) -> String {
1135 let mut record: std::collections::BTreeMap<String, String> = FIXED_RULE_FILES
1136 .iter()
1137 .map(|rel| (rel.to_string(), rule_file_hash(root, rel)))
1138 .collect();
1139 for rel in dirty {
1140 if is_rule_file(rel) {
1141 record.insert(rel.clone(), rule_file_hash(root, rel));
1142 }
1143 }
1144 serde_json::to_string(&record).unwrap_or_default()
1145}
1146
1147fn holds_resolver_config(dir: &Path) -> bool {
1150 ignore::WalkBuilder::new(dir)
1151 .standard_filters(false)
1152 .build()
1153 .flatten()
1154 .any(|e| {
1155 e.file_name()
1156 .to_str()
1157 .is_some_and(crate::dependency_resolve::is_resolver_config_name)
1158 })
1159}
1160
1161struct InvalidateOnDrop(std::path::PathBuf);
1164
1165impl Drop for InvalidateOnDrop {
1166 fn drop(&mut self) {
1167 crate::query::invalidate_caches(&self.0);
1168 }
1169}
1170
1171impl Indexer {
1172 pub fn new(cache: CacheManager, config: IndexConfig) -> Self {
1174 Self {
1175 cache,
1176 config,
1177 batch_limits: None,
1178 merge_limits: None,
1179 recent_limits: None,
1180 abort_at: None,
1181 rules_pending: std::sync::Mutex::new(None),
1182 }
1183 }
1184
1185 #[doc(hidden)]
1189 pub fn set_batch_limits(&mut self, max_files: usize, max_bytes: u64) {
1190 self.batch_limits = Some((max_files, max_bytes));
1191 }
1192
1193 #[doc(hidden)]
1196 pub fn set_merge_limits(&mut self, max_files: usize, max_bytes: u64) {
1197 self.merge_limits = Some((max_files, max_bytes));
1198 }
1199
1200 #[doc(hidden)]
1205 pub fn set_abort_point(&mut self, point: &str) {
1206 self.abort_at = Some(point.to_string());
1207 }
1208
1209 fn abort_point(&self, point: &str) {
1210 if self.abort_at.as_deref() == Some(point) {
1211 std::process::exit(ABORT_EXIT_CODE);
1213 }
1214 }
1215
1216 #[doc(hidden)]
1219 pub fn set_recent_limits(&mut self, max_files: usize, max_bytes: u64) {
1220 self.recent_limits = Some((max_files, max_bytes));
1221 }
1222
1223 fn recent_limits(&self, max_files: usize, max_bytes: u64) -> (usize, u64) {
1224 self.recent_limits.unwrap_or((
1225 RECENT_MAX_FILES.min(max_files),
1226 max_bytes / RECENT_BYTES_DIVISOR,
1227 ))
1228 }
1229
1230 fn merge_limits(&self, live_corpus_bytes: u64) -> (usize, u64) {
1231 self.merge_limits.unwrap_or((
1232 DELTA_MAX_FILES,
1233 live_corpus_bytes * DELTA_MAX_CORPUS_PERCENT / 100,
1234 ))
1235 }
1236
1237 fn batch_limits(&self) -> (usize, u64) {
1238 if let Some(limits) = self.batch_limits {
1239 return limits;
1240 }
1241 let files = std::env::var("REFLEX_INDEX_BATCH_FILES")
1242 .ok()
1243 .and_then(|v| v.parse::<usize>().ok())
1244 .filter(|&n| n > 0)
1245 .unwrap_or(BATCH_MAX_FILES);
1246 let bytes = std::env::var("REFLEX_INDEX_BATCH_BYTES")
1247 .ok()
1248 .and_then(|v| v.parse::<u64>().ok())
1249 .filter(|&n| n > 0)
1250 .unwrap_or(BATCH_MAX_BYTES);
1251 (files, bytes)
1252 }
1253
1254 pub fn path_policy(&self, root: &Path) -> PathPolicy {
1256 PathPolicy::from_config(root, &self.config)
1257 }
1258
1259 pub fn index(&self, root: impl AsRef<Path>, show_progress: bool) -> Result<IndexStats> {
1261 self.index_with_callback(root, show_progress, None)
1262 }
1263
1264 pub fn update_paths(&self, root: impl AsRef<Path>, paths: &[PathBuf]) -> Result<IndexStats> {
1276 let root = root.as_ref();
1277 match self.try_update_paths(root, paths)? {
1278 Some(stats) => Ok(stats),
1279 None => {
1280 log::info!("update_paths: the change needs a full index run");
1281 self.index(root, false)
1282 }
1283 }
1284 }
1285
1286 #[doc(hidden)]
1289 pub fn try_update_paths(&self, root: &Path, paths: &[PathBuf]) -> Result<Option<IndexStats>> {
1290 let started = Instant::now();
1291 let mut laps = Laps::new();
1292 let cache_dir = self.cache.path().to_path_buf();
1293 if !cache_dir.join(crate::cache::META_DB).exists() {
1294 return Ok(None);
1295 }
1296 let abs_root = root.canonicalize().unwrap_or_else(|_| root.to_path_buf());
1297 let mut named: Vec<String> = Vec::new();
1298 for path in paths {
1299 match classify_named(root, &abs_root, path) {
1300 Named::Skip => {}
1301 Named::Full => return Ok(None),
1302 Named::Path(rel) => named.push(rel),
1303 }
1304 }
1305 named.sort();
1306 named.dedup();
1307
1308 let in_git = crate::git::is_git_repo(root);
1310 let spawn_status = |specs: Vec<String>| {
1311 let root = root.to_path_buf();
1312 std::thread::spawn(move || {
1313 if specs.len() > 1000 {
1314 crate::git::changed_paths(&root)
1315 } else {
1316 let refs: Vec<&str> = specs.iter().map(String::as_str).collect();
1317 crate::git::changed_paths_in(&root, &refs)
1318 }
1319 })
1320 };
1321 let mut status_jobs = Vec::new();
1322 if in_git && !named.is_empty() {
1323 status_jobs.push(spawn_status(named.clone()));
1324 }
1325
1326 let _index_lock = crate::atomic_write::IndexLock::acquire_with_timeout(
1328 &cache_dir,
1329 std::time::Duration::from_secs(self.config.lock_wait_secs),
1330 )?;
1331 crate::atomic_write::remove_stale_tmp(&cache_dir);
1332 crate::query::invalidate_caches(root);
1333 let _invalidate_on_exit = InvalidateOnDrop(root.to_path_buf());
1334 self.yield_to_symbol_pass(&cache_dir)?;
1335 laps.lap("lock");
1336 self.cache.assert_writable(false)?;
1337 if !self.cache.check_schema_hash().unwrap_or(false)
1338 || !self.cache.check_extraction_hash().unwrap_or(false)
1339 {
1340 return Ok(None);
1341 }
1342 let run_start = self.run_marker(&cache_dir);
1343 laps.lap("checks");
1344
1345 let conn = crate::cache::open_meta_db(cache_dir.join(crate::cache::META_DB))?;
1346 let file_count = crate::meta_update::count_files(&conn)?;
1347 let stored_digest = crate::meta_update::get_statistic(&conn, RESOLVER_DIGEST_KEY)?;
1348 let meta_generation = crate::meta_update::get_statistic(&conn, INDEX_GENERATION_KEY)?
1349 .and_then(|g| g.parse::<u64>().ok());
1350 let synced_branch =
1351 crate::meta_update::get_statistic(&conn, crate::meta_update::SYNCED_BRANCH_KEY)?;
1352 let (Some(generation), Ok(indexed)) =
1353 (meta_generation, CacheManager::latest_branch_info_on(&conn))
1354 else {
1355 return Ok(None);
1356 };
1357 if !self.stores_intact(file_count, meta_generation) {
1358 return Ok(None);
1359 }
1360 let generation = generation + 1;
1361 laps.lap("snapshot");
1362
1363 let Some(resolver_configs) = crate::dependency_resolve::ResolverConfigs::from_saved_list(
1365 root,
1366 &cache_dir,
1367 &self.config.vendored_patterns,
1368 ) else {
1369 return Ok(None);
1370 };
1371 if stored_digest.as_deref() != Some(resolver_configs.digest.as_str()) {
1372 return Ok(None);
1373 }
1374 laps.lap("configs");
1375
1376 let (branch, commit) = if in_git {
1378 let Ok((commit, branch)) = crate::git::head_commit_and_branch(root) else {
1379 return Ok(None);
1380 };
1381 (branch, Some(commit))
1382 } else {
1383 ("_default".to_string(), None)
1384 };
1385 if synced_branch.as_deref() != Some(branch.as_str()) || branch != indexed.branch {
1388 return Ok(None);
1389 }
1390 if let Some(commit) = &commit
1391 && *commit != indexed.commit_sha
1392 {
1393 let Ok(moved) = crate::git::diff_names(root, &indexed.commit_sha, commit) else {
1394 return Ok(None);
1395 };
1396 let mut extra: Vec<String> = Vec::new();
1397 for path in moved {
1398 match classify_named(root, &abs_root, Path::new(&path)) {
1399 Named::Skip => {}
1400 Named::Full => return Ok(None),
1401 Named::Path(rel) => {
1402 if named.binary_search(&rel).is_err() {
1403 extra.push(rel);
1404 }
1405 }
1406 }
1407 }
1408 if !extra.is_empty() {
1409 status_jobs.push(spawn_status(extra.clone()));
1410 named.extend(extra);
1411 named.sort();
1412 named.dedup();
1413 }
1414 }
1415 if resolver_configs.lists_any_under(root, &named)
1416 || named.iter().any(|rel| {
1417 let dir = root.join(rel);
1418 dir.is_dir() && holds_resolver_config(&dir)
1419 })
1420 {
1421 return Ok(None);
1422 }
1423 let targets = Arc::new(Targets::new(&named));
1424 laps.lap("git");
1425
1426 let stored = crate::meta_update::load_rows_under(&conn, &named)?;
1429 let Discovered {
1430 rels: found_rels,
1431 metas: found_metas,
1432 skipped_too_large,
1433 skipped_bytes_too_large,
1434 skipped_binary,
1435 ..
1436 } = self.discover_files_in(root, &stored, Some(Arc::clone(&targets)))?;
1437 let found: HashMap<&str, usize> = found_rels
1438 .iter()
1439 .enumerate()
1440 .map(|(i, rel)| (rel.as_str(), i))
1441 .collect();
1442 laps.lap("walk");
1443
1444 let num_threads = crate::models::resolve_thread_count(self.config.parallel_threads, 32);
1446 let pool = rayon::ThreadPoolBuilder::new()
1447 .num_threads(num_threads)
1448 .build()
1449 .context("Failed to create thread pool")?;
1450 let to_hash: Vec<usize> = (0..found_rels.len())
1451 .filter(|&i| {
1452 stored.get(&found_rels[i]).is_some_and(|row| {
1453 !found_metas[i]
1454 .as_ref()
1455 .is_some_and(|md| row.stat_matches(md))
1456 })
1457 })
1458 .collect();
1459 let hashed: Vec<(usize, Option<String>)> = pool.install(|| {
1460 to_hash
1461 .par_iter()
1462 .map(|&i| {
1463 let hash = std::fs::read(root.join(&found_rels[i]))
1464 .ok()
1465 .map(|b| self.hash_content(&b));
1466 (i, hash)
1467 })
1468 .collect()
1469 });
1470 let mut found_status: Vec<FileStatus> = found_rels
1471 .iter()
1472 .map(|rel| {
1473 if stored.contains_key(rel) {
1474 FileStatus::Unchanged
1475 } else {
1476 FileStatus::Added
1477 }
1478 })
1479 .collect();
1480 for (i, hash) in hashed {
1481 found_status[i] = match hash {
1482 Some(h) if h == stored[&found_rels[i]].hash => FileStatus::Touched,
1483 _ => FileStatus::Modified,
1484 };
1485 }
1486 let rewritten =
1487 |i: usize| matches!(found_status[i], FileStatus::Added | FileStatus::Modified);
1488 let deleted: Vec<&str> = {
1490 let mut d: Vec<&str> = stored
1491 .keys()
1492 .map(String::as_str)
1493 .filter(|p| !found.contains_key(p))
1494 .collect();
1495 d.sort_unstable();
1496 d
1497 };
1498 laps.lap("hash");
1499
1500 let to_place: Vec<(&str, Option<&crate::meta_update::StoredFile>)> = (0..found_rels.len())
1502 .filter(|&i| rewritten(i))
1503 .map(|i| (found_rels[i].as_str(), stored.get(&found_rels[i])))
1504 .collect();
1505 let deleted_ids: std::collections::HashSet<i64> =
1506 deleted.iter().map(|p| stored[*p].id).collect();
1507 let mut walk_order = crate::walk_order::WalkOrder::new(root);
1508 let Some(seq_of) = place_rewrites(&conn, &mut walk_order, &to_place, &deleted_ids)? else {
1509 return Ok(None);
1510 };
1511 drop(conn);
1512 let added = found_status
1513 .iter()
1514 .filter(|s| **s == FileStatus::Added)
1515 .count();
1516 let live_files = file_count - deleted.len() + added;
1517 laps.lap("order");
1518
1519 let mut rewrites = Vec::new();
1520 let mut rewrite_rows = HashMap::new();
1521 for (i, rel) in found_rels.iter().enumerate() {
1522 if rewritten(i) {
1523 rewrites.push(Rewrite {
1524 rel: rel.clone(),
1525 size: found_metas[i].as_ref().map_or(0, |md| md.size()),
1526 walk_seq: seq_of[rel],
1527 });
1528 if let Some(row) = stored.get(rel) {
1529 rewrite_rows.insert(rel.clone(), row.clone());
1530 }
1531 }
1532 }
1533 let dirty_job = || -> DirtyFlags {
1537 let mut now_dirty: std::collections::HashSet<String> = std::collections::HashSet::new();
1538 let mut ok = true;
1539 for job in status_jobs {
1540 match job.join() {
1541 Ok(Ok(paths)) => now_dirty.extend(paths),
1542 _ => ok = false,
1543 }
1544 }
1545 let is_dirty = |rel: &str| !ok || now_dirty.contains(rel);
1546 let mut flags = DirtyFlags {
1547 rewrites: std::collections::HashSet::new(),
1548 touched: Vec::new(),
1549 flips: Vec::new(),
1550 git_dirty: in_git && (indexed.is_dirty || !ok || !now_dirty.is_empty()),
1551 };
1552 for (i, rel) in found_rels.iter().enumerate() {
1553 let dirty = is_dirty(rel);
1554 if rewritten(i) {
1555 if dirty {
1556 flags.rewrites.insert(rel.clone());
1557 }
1558 } else if let Some(row) = stored.get(rel) {
1559 if found_status[i] == FileStatus::Touched {
1560 let (size, mtime) = found_metas[i]
1561 .as_ref()
1562 .map(|md| (md.size(), md.recorded_mtime_ns(run_start)))
1563 .unwrap_or((0, 0));
1564 flags.touched.push((row.id, size, mtime, dirty));
1565 } else if dirty != row.dirty {
1566 flags.flips.push((row.id, dirty));
1567 }
1568 }
1569 }
1570 flags
1571 };
1572 let branch_hashes: HashMap<String, String> = {
1575 let conn = crate::cache::open_meta_db(cache_dir.join(crate::cache::META_DB))?;
1576 let mut stmt = conn.prepare(
1577 "SELECT fb.hash FROM file_branches fb
1578 JOIN files f ON f.id = fb.file_id
1579 JOIN branches b ON b.id = fb.branch_id
1580 WHERE b.name = ? AND f.path = ?",
1581 )?;
1582 let mut out = HashMap::new();
1583 for r in &rewrites {
1584 let hash: Option<String> = rusqlite::OptionalExtension::optional(
1585 stmt.query_row(rusqlite::params![branch, r.rel], |row| row.get(0)),
1586 )?;
1587 if let Some(hash) = hash {
1588 out.insert(r.rel.clone(), hash);
1589 }
1590 }
1591 out
1592 };
1593 let skipped = (skipped_too_large, skipped_bytes_too_large, skipped_binary);
1594 let rewritten_count = rewrites.len();
1595 laps.lap("change_set");
1596
1597 let result = if deleted.is_empty() && rewrites.is_empty() {
1598 let flags = dirty_job();
1599 self.refresh_named(RefreshNamed {
1600 touched: &flags.touched,
1601 flips: &flags.flips,
1602 walk_moves: &[],
1603 branch: &branch,
1604 commit: commit.as_deref(),
1605 git_dirty: flags.git_dirty,
1606 live_files,
1607 skipped,
1608 })
1609 .map(Some)
1610 } else {
1611 let deleted_file_count = deleted.iter().filter(|p| !root.join(p).exists()).count();
1612 self.publish_delta(DeltaChanges {
1613 root,
1614 rewrites,
1615 deleted: deleted
1616 .iter()
1617 .map(|p| (p.to_string(), stored[*p].id))
1618 .collect(),
1619 walk_moves: Vec::new(),
1620 dirty: Box::new(dirty_job),
1621 stored: rewrite_rows,
1622 sync_all_branch_rows: false,
1623 full_stats: false,
1624 live_files,
1625 breakdown_rest: (0, 0, live_files - rewritten_count),
1626 branch_hashes,
1627 branch: &branch,
1628 commit: commit.as_deref(),
1629 resolver_configs: &resolver_configs,
1630 run_start,
1631 generation,
1632 pool: &pool,
1633 deleted_file_count,
1634 skipped,
1635 })
1636 };
1637 laps.lap("publish");
1638 log::info!(
1639 "update_paths: {} named paths, {} rewritten, {} deleted in {} ms;{}",
1640 named.len(),
1641 rewritten_count,
1642 deleted.len(),
1643 started.elapsed().as_millis(),
1644 laps.line
1645 );
1646 result
1647 }
1648
1649 const SYMBOL_PASS_YIELD_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);
1654
1655 fn yield_to_symbol_pass(&self, cache_dir: &Path) -> Result<()> {
1667 use crate::background_indexer::BackgroundIndexer;
1668
1669 let Some(holder) = BackgroundIndexer::lock_holder(cache_dir) else {
1670 BackgroundIndexer::clear_cancel(cache_dir);
1672 return Ok(());
1673 };
1674
1675 log::info!(
1676 "Symbol indexing is running (pid {}); asking it to yield",
1677 holder.pid
1678 );
1679 let _ = BackgroundIndexer::request_cancel(cache_dir);
1680
1681 let deadline = std::time::Instant::now() + Self::SYMBOL_PASS_YIELD_TIMEOUT;
1682 let mut pause = std::time::Duration::from_millis(5);
1685 while std::time::Instant::now() < deadline {
1686 if !BackgroundIndexer::is_running(cache_dir) {
1687 BackgroundIndexer::clear_cancel(cache_dir);
1688 log::info!("Symbol indexing yielded; continuing");
1689 return Ok(());
1690 }
1691 std::thread::sleep(pause);
1692 pause = (pause * 2).min(std::time::Duration::from_millis(100));
1693 }
1694
1695 BackgroundIndexer::clear_cancel(cache_dir);
1697 let status = BackgroundIndexer::get_status(cache_dir).ok().flatten();
1698 Err(crate::errors::ReflexError::SymbolIndexingInProgress {
1699 pid: holder.pid,
1700 started_at: holder.started_clock(),
1701 processed: status.as_ref().map(|s| s.processed_files).unwrap_or(0),
1702 total: status.as_ref().map(|s| s.total_files).unwrap_or(0),
1703 }
1704 .into())
1705 }
1706
1707 pub fn index_with_callback(
1709 &self,
1710 root: impl AsRef<Path>,
1711 show_progress: bool,
1712 progress_callback: Option<ProgressCallback>,
1713 ) -> Result<IndexStats> {
1714 let root = root.as_ref();
1715 let stats = self.index_run(root, show_progress, progress_callback)?;
1716 let pending = self
1720 .rules_pending
1721 .lock()
1722 .unwrap_or_else(|e| e.into_inner())
1723 .take();
1724 if let Some(record) = pending {
1725 let conn = crate::cache::open_meta_db(self.cache.path().join(crate::cache::META_DB))?;
1726 crate::meta_update::set_statistic(
1727 &conn,
1728 RULE_FILES_KEY,
1729 &record,
1730 chrono::Utc::now().timestamp(),
1731 )?;
1732 crate::query::invalidate_caches(root);
1733 }
1734 Ok(stats)
1735 }
1736
1737 fn index_run(
1738 &self,
1739 root: &Path,
1740 show_progress: bool,
1741 progress_callback: Option<ProgressCallback>,
1742 ) -> Result<IndexStats> {
1743 log::info!("Indexing directory: {:?}", root);
1744 let mut laps = Laps::new();
1745
1746 let cache_dir = self.cache.path().to_path_buf();
1750 let _index_lock = crate::atomic_write::IndexLock::acquire_with_timeout(
1751 &cache_dir,
1752 std::time::Duration::from_secs(self.config.lock_wait_secs),
1753 )?;
1754 crate::atomic_write::remove_stale_tmp(&cache_dir);
1756
1757 crate::query::invalidate_caches(root);
1761 let _invalidate_on_exit = InvalidateOnDrop(root.to_path_buf());
1762
1763 self.yield_to_symbol_pass(&cache_dir)?;
1768 laps.lap("lock");
1769
1770 let num_threads = crate::models::resolve_thread_count(self.config.parallel_threads, 32);
1776
1777 log::info!(
1778 "Using {} threads for parallel indexing (out of {} available)",
1779 num_threads,
1780 std::thread::available_parallelism()
1781 .map(|n| n.get())
1782 .unwrap_or(4)
1783 );
1784
1785 self.cache.assert_writable(false)?;
1791
1792 let schema_ok = self.cache.check_schema_hash().unwrap_or(false);
1796 let extraction_ok = self.cache.check_extraction_hash().unwrap_or(false);
1797
1798 if schema_ok && extraction_ok {
1801 self.cache.ensure_config()?;
1802 } else {
1803 self.cache.init()?;
1804 }
1805
1806 if !schema_ok || !extraction_ok {
1810 match self.cache.clear_symbol_cache() {
1811 Ok(n) if n > 0 => {
1812 log::info!("Cleared {} symbol cache rows written by other code", n)
1813 }
1814 Ok(_) => {}
1815 Err(e) => log::warn!("Failed to clear the symbol cache: {}", e),
1816 }
1817 }
1818
1819 let run_start = self.run_marker(&cache_dir);
1825
1826 laps.lap("init");
1827
1828 let pool = rayon::ThreadPoolBuilder::new()
1829 .num_threads(num_threads)
1830 .build()
1831 .context("Failed to create thread pool")?;
1832
1833 let meta_path = cache_dir.join(crate::cache::META_DB);
1837 let phase_start = Instant::now();
1838 let (git_state, resolver_configs, meta_read, walked) = std::thread::scope(|scope| {
1839 let git = scope.spawn(|| {
1840 let t = Instant::now();
1841 let r = crate::git::get_git_state_optional(root);
1842 (r, t.elapsed().as_millis())
1843 });
1844 let configs = scope.spawn(|| {
1845 let t = Instant::now();
1846 let r = crate::dependency_resolve::ResolverConfigs::discover(
1847 root,
1848 &self.config.vendored_patterns,
1849 );
1850 (r, t.elapsed().as_millis())
1851 });
1852 let disk = scope.spawn(|| self.check_disk_space(root));
1854 let meta_read = scope.spawn(|| -> Result<_> {
1855 let t = Instant::now();
1856 let conn = crate::cache::open_meta_db(&meta_path)?;
1857 let rows = crate::meta_update::load_stored_files(&conn)?;
1858 log::debug!("meta.db rows read in {} ms", t.elapsed().as_millis());
1859 Ok((
1860 rows,
1861 crate::meta_update::get_statistic(&conn, RESOLVER_DIGEST_KEY)?,
1862 crate::meta_update::get_statistic(&conn, INDEX_GENERATION_KEY)?
1863 .and_then(|g| g.parse::<u64>().ok()),
1864 crate::meta_update::get_statistic(
1865 &conn,
1866 crate::meta_update::SYNCED_BRANCH_KEY,
1867 )?,
1868 crate::meta_update::get_statistic(&conn, RULE_FILES_KEY)?,
1869 ))
1870 });
1871 let t = Instant::now();
1872 let walked = self.walk_candidates(root, None);
1873 let walk_ms = t.elapsed().as_millis();
1874 let (git, configs) = (git.join(), configs.join());
1875 log::info!(
1876 "discovery phase: walk {} ms, git {} ms, resolver configs {} ms",
1877 walk_ms,
1878 git.as_ref().map_or(0, |g| g.1),
1879 configs.as_ref().map_or(0, |c| c.1)
1880 );
1881 let disk = disk
1882 .join()
1883 .unwrap_or_else(|_| Err(anyhow::anyhow!("disk space check panicked")));
1884 (
1885 git.map(|g| g.0),
1886 configs.map(|c| c.0),
1887 meta_read.join(),
1888 walked.and_then(|w| disk.map(|()| w)),
1889 )
1890 });
1891 let (stored, stored_digest, meta_generation, synced_branch, stored_rules) =
1892 meta_read.map_err(|_| anyhow::anyhow!("meta.db read thread panicked"))??;
1893 let discovered = walked.map(|w| Self::drop_binaries(root, w, &stored, Some(&pool)));
1896 let prev_manifest = crate::snapshot::read_manifest(&cache_dir).ok().flatten();
1898 let generation = prev_manifest
1899 .as_ref()
1900 .map(|m| m.generation)
1901 .max(meta_generation)
1902 .unwrap_or(0)
1903 + 1;
1904 let git_state = git_state.map_err(|_| anyhow::anyhow!("git state thread panicked"))??;
1905 let resolver_configs =
1906 resolver_configs.map_err(|_| anyhow::anyhow!("resolver config walk panicked"))?;
1907 if let Err(e) = resolver_configs.save_list(root, &cache_dir) {
1909 log::warn!("Failed to save the resolver config list: {:#}", e);
1910 }
1911 let Discovered {
1912 rels,
1913 sizes,
1914 metas,
1915 skipped_too_large,
1916 skipped_bytes_too_large,
1917 skipped_binary,
1918 } = discovered?;
1919 let total_files = rels.len();
1920 laps.lap("discover");
1921 log::info!(
1922 "Discovered {} files to index ({} skipped: too large, {} binary) in {} ms",
1923 total_files,
1924 skipped_too_large,
1925 skipped_binary,
1926 phase_start.elapsed().as_millis()
1927 );
1928
1929 let branch = git_state
1930 .as_ref()
1931 .map(|s| s.branch.clone())
1932 .unwrap_or_else(|| "_default".to_string());
1933 if let Some(ref state) = git_state {
1934 log::info!(
1935 "Git state: branch='{}', commit='{}', dirty={}",
1936 state.branch,
1937 state.commit,
1938 state.dirty
1939 );
1940 } else {
1941 log::info!("Not a git repository, using default branch");
1942 }
1943 let commit = git_state.as_ref().map(|s| s.commit.clone());
1944 let dirty_paths: std::collections::HashSet<String> = git_state
1945 .as_ref()
1946 .map(|s| s.dirty_paths.clone())
1947 .unwrap_or_default();
1948 let rules = rule_files_record(root, &dirty_paths);
1949 *self.rules_pending.lock().unwrap_or_else(|e| e.into_inner()) =
1950 (stored_rules.as_deref() != Some(rules.as_str())).then_some(rules);
1951
1952 let in_sync = synced_branch.as_deref() == Some(branch.as_str());
1955 let existing_hashes = if in_sync {
1956 BranchHashes::Synced(&stored)
1957 } else {
1958 BranchHashes::Loaded(self.cache.load_hashes_for_branch(&branch)?)
1959 };
1960 laps.lap("branch_hashes");
1961
1962 let classify_start = Instant::now();
1965 let rows_of: Vec<Option<&crate::meta_update::StoredFile>> =
1967 rels.iter().map(|rel| stored.get(rel)).collect();
1968 let to_hash: Vec<usize> = (0..total_files)
1969 .filter(|&i| {
1970 rows_of[i]
1971 .is_some_and(|row| !metas[i].as_ref().is_some_and(|md| row.stat_matches(md)))
1972 })
1973 .collect();
1974 let hashed: Vec<(usize, Option<String>)> = pool.install(|| {
1975 to_hash
1976 .par_iter()
1977 .map(|&i| {
1978 (
1979 i,
1980 std::fs::read(root.join(&rels[i]))
1981 .ok()
1982 .map(|b| self.hash_content(&b)),
1983 )
1984 })
1985 .collect()
1986 });
1987 let mut status: Vec<FileStatus> = rows_of
1988 .iter()
1989 .map(|row| {
1990 if row.is_some() {
1991 FileStatus::Unchanged
1992 } else {
1993 FileStatus::Added
1994 }
1995 })
1996 .collect();
1997 for (i, hash) in hashed {
1998 status[i] = match (hash, rows_of[i]) {
1999 (Some(h), Some(row)) if h == row.hash => FileStatus::Touched,
2000 _ => FileStatus::Modified,
2001 };
2002 }
2003 let walked_with_row = status.iter().filter(|s| **s != FileStatus::Added).count();
2006 let gone: Vec<&String> = if walked_with_row == stored.len() {
2007 Vec::new()
2008 } else {
2009 let discovered_set: std::collections::HashSet<&str> =
2010 rels.iter().map(String::as_str).collect();
2011 stored
2012 .keys()
2013 .filter(|p| !discovered_set.contains(p.as_str()))
2014 .collect()
2015 };
2016 let deleted_file_count = gone.iter().filter(|p| !root.join(p).exists()).count();
2019 let added = status.iter().filter(|s| **s == FileStatus::Added).count();
2020 let modified = status
2021 .iter()
2022 .filter(|s| **s == FileStatus::Modified)
2023 .count();
2024 log::info!(
2025 "phase classify: {} added, {} modified, {} touched, {} gone; {} files hashed in {} ms",
2026 added,
2027 modified,
2028 status.iter().filter(|s| **s == FileStatus::Touched).count(),
2029 gone.len(),
2030 to_hash.len(),
2031 classify_start.elapsed().as_millis()
2032 );
2033
2034 let full_deps = !schema_ok
2037 || !extraction_ok
2038 || stored_digest.as_deref() != Some(resolver_configs.digest.as_str());
2039 let content_changed = added > 0 || modified > 0 || !gone.is_empty();
2040 laps.lap("classify");
2041 let stores_ok = schema_ok && self.stores_intact(stored.len(), meta_generation);
2042 laps.lap("snapshot");
2043
2044 if !content_changed && !full_deps && stores_ok {
2045 log::info!("No files changed - skipping index rebuild");
2046 log::info!("index run phases:{}", laps.line);
2047 return self.refresh_unchanged(RefreshUnchanged {
2048 rels: &rels,
2049 metas: &metas,
2050 status: &status,
2051 rows: &rows_of,
2052 existing_hashes: &existing_hashes,
2053 dirty_paths: &dirty_paths,
2054 branch: &branch,
2055 in_sync,
2056 commit: commit.as_deref(),
2057 git_dirty: git_state.as_ref().map(|s| s.dirty),
2058 run_start,
2059 skipped: (skipped_too_large, skipped_bytes_too_large, skipped_binary),
2060 });
2061 }
2062 if content_changed && !full_deps && stores_ok {
2063 let seqs = crate::meta_update::plan_walk_seq(
2066 &rels
2067 .iter()
2068 .map(|rel| stored.get(rel).map(|s| s.walk_seq))
2069 .collect::<Vec<_>>(),
2070 );
2071 let mut rewrites = Vec::new();
2072 let mut dirty_rewrites = std::collections::HashSet::new();
2073 let mut rewrite_rows = HashMap::new();
2074 let mut branch_hashes = HashMap::new();
2075 let (mut touched, mut walk_moves, mut flips) = (Vec::new(), Vec::new(), Vec::new());
2076 let mut rest: Vec<(&str, &str)> = Vec::new();
2077 for i in 0..total_files {
2078 let rel = &rels[i];
2079 let dirty = dirty_paths.contains(rel);
2080 match (status[i], stored.get(rel)) {
2081 (FileStatus::Added | FileStatus::Modified, row) => {
2082 rewrites.push(Rewrite {
2083 rel: rel.clone(),
2084 size: sizes[i],
2085 walk_seq: seqs[i],
2086 });
2087 if dirty {
2088 dirty_rewrites.insert(rel.clone());
2089 }
2090 if let Some(row) = row {
2091 rewrite_rows.insert(rel.clone(), row.clone());
2092 }
2093 if let Some(h) = existing_hashes.get(rel) {
2094 branch_hashes.insert(rel.clone(), h.to_string());
2095 }
2096 }
2097 (_, Some(row)) => {
2098 if seqs[i] != row.walk_seq {
2099 walk_moves.push((row.id, seqs[i]));
2100 }
2101 if status[i] == FileStatus::Touched {
2102 let (size, mtime) = metas[i]
2103 .as_ref()
2104 .map(|md| (md.size(), md.recorded_mtime_ns(run_start)))
2105 .unwrap_or((0, 0));
2106 touched.push((row.id, size, mtime, dirty));
2107 } else if dirty != row.dirty {
2108 flips.push((row.id, dirty));
2109 }
2110 rest.push((rel.as_str(), row.hash.as_str()));
2111 }
2112 (_, None) => unreachable!("an unchanged file has a row"),
2113 }
2114 }
2115 let git_dirty = git_state.as_ref().map(|s| s.dirty).unwrap_or(false);
2116 let changes = DeltaChanges {
2117 root,
2118 rewrites,
2119 deleted: gone.iter().map(|p| ((*p).clone(), stored[*p].id)).collect(),
2120 walk_moves,
2121 dirty: Box::new(move || DirtyFlags {
2122 rewrites: dirty_rewrites,
2123 touched,
2124 flips,
2125 git_dirty,
2126 }),
2127 stored: rewrite_rows,
2128 sync_all_branch_rows: !in_sync,
2129 full_stats: true,
2130 live_files: total_files,
2131 breakdown_rest: breakdown(rest.into_iter(), &existing_hashes),
2132 branch_hashes,
2133 branch: &branch,
2134 commit: commit.as_deref(),
2135 resolver_configs: &resolver_configs,
2136 run_start,
2137 generation,
2138 pool: &pool,
2139 deleted_file_count,
2140 skipped: (skipped_too_large, skipped_bytes_too_large, skipped_binary),
2141 };
2142 match self.publish_delta(changes)? {
2143 Some(stats) => return Ok(stats),
2144 None => log::info!("The delta would pass its limits - building a new base"),
2145 }
2146 }
2147 if full_deps {
2148 log::info!(
2149 "Resolver configs, extraction code or schema changed - re-extracting every file's imports"
2150 );
2151 }
2152
2153 let mut files_indexed = 0;
2157 let mut indexed: Vec<IndexedFile> = Vec::with_capacity(total_files);
2158
2159 let mut trigram_builder = TrigramIndexBuilder::new(self.cache.path().join("trigram_temp"));
2163 let mut content_writer = ContentWriter::new();
2164
2165 let (content_name, trigrams_name, plan_name) = crate::snapshot::base_file_names(generation);
2167 let content_path = cache_dir.join(&content_name);
2168 content_writer
2169 .init(content_path.clone())
2170 .context("Failed to initialize content writer")?;
2171
2172 let pb = if show_progress {
2174 let pb = ProgressBar::new(total_files as u64);
2175 pb.set_draw_target(ProgressDrawTarget::stderr());
2176 pb.set_style(
2177 ProgressStyle::default_bar()
2178 .template("[{elapsed_precise}] [{bar:40.cyan/blue}] {pos}/{len} files ({percent}%) {msg}")
2179 .unwrap()
2180 .progress_chars("=>-")
2181 );
2182 pb.enable_steady_tick(std::time::Duration::from_millis(100));
2184 pb
2185 } else {
2186 ProgressBar::hidden()
2187 };
2188
2189 let progress_counter = Arc::new(AtomicU64::new(0));
2191 let progress_status = Arc::new(Mutex::new("Indexing files...".to_string()));
2193
2194 let batch_phase_start = Instant::now();
2195 let mut pool_ms = 0u128;
2196 let mut flush_ms = 0u128;
2197
2198 let merge_source: Option<(crate::snapshot::IndexSnapshot, HashMap<String, u32>)> =
2203 if stores_ok {
2204 crate::snapshot::IndexSnapshot::open(&cache_dir)
2205 .ok()
2206 .map(|snapshot| {
2207 let ids = snapshot
2208 .live_ids()
2209 .filter_map(|id| {
2210 Some((snapshot.get_file_path(id)?.to_str()?.to_string(), id))
2211 })
2212 .collect();
2213 (snapshot, ids)
2214 })
2215 } else {
2216 None
2217 };
2218 let from_snapshot = std::sync::atomic::AtomicUsize::new(0);
2219
2220 let counter_for_thread = Arc::clone(&progress_counter);
2222 let status_for_thread = Arc::clone(&progress_status);
2223 let pb_clone = pb.clone();
2224 let callback_for_thread = progress_callback.clone();
2225 let total_files_for_thread = total_files;
2226 let progress_thread = if show_progress || callback_for_thread.is_some() {
2227 Some(std::thread::spawn(move || {
2228 loop {
2229 let count = counter_for_thread.load(Ordering::Relaxed);
2230 pb_clone.set_position(count);
2231
2232 if let Some(ref callback) = callback_for_thread {
2234 let status = status_for_thread.lock().unwrap().clone();
2235 callback(count as usize, total_files_for_thread, status);
2236 }
2237
2238 if count >= total_files_for_thread as u64 {
2239 break;
2240 }
2241 std::thread::sleep(std::time::Duration::from_millis(50));
2242 }
2243 }))
2244 } else {
2245 None
2246 };
2247
2248 let (max_files, max_bytes) = self.batch_limits();
2253 let batches = plan_batches(&sizes, max_files, max_bytes);
2254 let num_batches = batches.len();
2255 let spill_to_disk = num_batches > 1;
2256 log::info!(
2257 "Processing {} files in {} batches (<= {} files, ~{} MB each)",
2258 total_files,
2259 num_batches,
2260 max_files,
2261 max_bytes >> 20
2262 );
2263
2264 for (batch_idx, batch_range) in batches.into_iter().enumerate() {
2265 let batch_start = batch_range.start;
2266 let batch_len = batch_range.len();
2267 log::info!(
2268 "Processing batch {}/{} ({} files)",
2269 batch_idx + 1,
2270 num_batches,
2271 batch_len
2272 );
2273 let pool_start = Instant::now();
2274
2275 let counter_clone = Arc::clone(&progress_counter);
2278 let tsconfigs = &resolver_configs.tsconfigs;
2279 let ctx = ProcessCtx {
2280 root,
2281 rels: &rels,
2282 stored: &stored,
2283 run_start,
2284 full_deps,
2285 tsconfigs,
2286 };
2287 let results: Vec<Option<FileProcessingResult>> = pool.install(|| {
2288 batch_range
2289 .into_par_iter()
2290 .map_init(Vec::<u64>::new, |trigram_scratch, i| {
2291 let reused = match &merge_source {
2292 Some((snapshot, ids))
2293 if matches!(
2294 status[i],
2295 FileStatus::Unchanged | FileStatus::Touched
2296 ) =>
2297 {
2298 ids.get(&rels[i]).and_then(|&id| {
2299 self.reuse_file(
2300 &ctx,
2301 snapshot,
2302 id,
2303 i,
2304 &metas[i],
2305 trigram_scratch,
2306 )
2307 })
2308 }
2309 _ => None,
2310 };
2311 let result = match reused {
2312 Some(r) => {
2313 from_snapshot.fetch_add(1, Ordering::Relaxed);
2314 Some(r)
2315 }
2316 None => self.process_file(&ctx, i, trigram_scratch),
2317 };
2318 counter_clone.fetch_add(1, Ordering::Relaxed);
2319 result
2320 })
2321 .collect()
2322 });
2323 pool_ms += pool_start.elapsed().as_millis();
2324 if let Some((snapshot, _)) = &merge_source {
2327 snapshot.release_pages();
2328 }
2329
2330 for (offset, result) in results.into_iter().enumerate() {
2332 let Some(result) = result else { continue };
2333 let i = batch_start + offset;
2334 let normalized_pathbuf = PathBuf::from(&rels[i]);
2338
2339 let _file_id =
2342 trigram_builder.add_file(normalized_pathbuf.clone(), result.trigram_run);
2343
2344 content_writer.add_file(normalized_pathbuf, &result.content);
2346
2347 files_indexed += 1;
2348 indexed.push(IndexedFile {
2349 index: i,
2350 hash: result.hash,
2351 language: result.language,
2352 line_count: result.line_count,
2353 size: result.size,
2354 mtime_ns: result.mtime_ns,
2355 imports: result.imports,
2356 });
2357 }
2358
2359 let flush_msg = format!(
2361 "Building trigram batch {}/{}...",
2362 batch_idx + 1,
2363 num_batches
2364 );
2365 if show_progress {
2366 pb.set_message(flush_msg.clone());
2367 }
2368 *progress_status.lock().unwrap() = flush_msg;
2369 let flush_start = Instant::now();
2370 trigram_builder
2371 .flush_batch(&pool, spill_to_disk)
2372 .context("Failed to build trigram batch")?;
2373 flush_ms += flush_start.elapsed().as_millis();
2374 }
2375 log::info!(
2376 "phase read+extract: {} ms in pool, {} ms building trigram batches, {} ms total; {} files from the published stores",
2377 pool_ms,
2378 flush_ms,
2379 batch_phase_start.elapsed().as_millis(),
2380 from_snapshot.load(Ordering::Relaxed)
2381 );
2382 drop(merge_source);
2383
2384 if let Some(thread) = progress_thread {
2386 let _ = thread.join();
2387 }
2388
2389 if show_progress {
2391 let final_count = progress_counter.load(Ordering::Relaxed);
2392 pb.set_position(final_count);
2393 }
2394
2395 *progress_status.lock().unwrap() = "Writing file metadata to database...".to_string();
2397 if show_progress {
2398 pb.set_message("Writing file metadata to database...".to_string());
2399 }
2400
2401 let (new_file_count, modified_file_count, unchanged_file_count) = breakdown(
2403 indexed
2404 .iter()
2405 .map(|f| (rels[f.index].as_str(), f.hash.as_str())),
2406 &existing_hashes,
2407 );
2408
2409 log::info!("Indexed {} files", files_indexed);
2410
2411 *progress_status.lock().unwrap() = "Writing trigram index...".to_string();
2417 if show_progress {
2418 pb.set_message("Writing trigram index...".to_string());
2419 }
2420 let trigrams_path = cache_dir.join(&trigrams_name);
2421 let write_start = Instant::now();
2422 trigram_builder
2423 .write_with_plan(&pool, &trigrams_path, Some(&cache_dir.join(&plan_name)))
2424 .context("Failed to write trigram index")?;
2425 log::info!(
2426 "phase trigram write: {} trigrams, {} files, {} ms",
2427 trigram_builder.trigram_count(),
2428 trigram_builder.file_count(),
2429 write_start.elapsed().as_millis()
2430 );
2431
2432 *progress_status.lock().unwrap() = "Finalizing content store...".to_string();
2434 if show_progress {
2435 pb.set_message("Finalizing content store...".to_string());
2436 }
2437 content_writer
2438 .finalize_if_needed()
2439 .context("Failed to finalize content store")?;
2440 log::info!(
2441 "Wrote {} files ({} bytes) to {}",
2442 content_writer.file_count(),
2443 content_writer.content_size(),
2444 content_name
2445 );
2446
2447 let manifest = crate::snapshot::Manifest::for_base(
2453 generation,
2454 crate::snapshot::SegmentFiles {
2455 content: content_name.clone(),
2456 trigrams: trigrams_name.clone(),
2457 plan: Some(plan_name.clone()),
2458 files: content_writer.file_count() as u64,
2459 content_bytes: std::fs::metadata(&content_path)?.len(),
2460 trigrams_bytes: std::fs::metadata(&trigrams_path)?.len(),
2461 },
2462 trigram_builder.trigram_count() as u64,
2463 content_writer.content_size() as u64,
2464 );
2465 self.abort_point("base-files");
2466 crate::snapshot::write_manifest(&cache_dir, &manifest)?;
2467 crate::snapshot::link_fixed_names(&cache_dir, &manifest);
2468 self.abort_point("base-manifest");
2469
2470 let meta_start = Instant::now();
2471 let present: Vec<usize> = indexed.iter().map(|f| f.index).collect();
2472 self.write_meta(MetaWrite {
2473 root,
2474 rels: &rels,
2475 present: &present,
2476 written: &indexed,
2477 metas: &metas,
2478 run_start,
2479 stored: &stored,
2480 status: &status,
2481 dirty_paths: &dirty_paths,
2482 branch: &branch,
2483 commit: commit.as_deref(),
2484 resolver_configs: &resolver_configs,
2485 full_deps,
2486 generation,
2487 })?;
2488 self.abort_point("base-meta");
2489
2490 self.cache.update_branch_metadata(
2492 &branch,
2493 commit.as_deref(),
2494 indexed.len(),
2495 git_state.as_ref().map(|s| s.dirty).unwrap_or(false),
2496 )?;
2497
2498 self.cache
2501 .checkpoint_wal()
2502 .context("Failed to checkpoint WAL")?;
2503 log::info!(
2504 "phase meta.db (files, branches, dependencies, exports): {} ms",
2505 meta_start.elapsed().as_millis()
2506 );
2507
2508 crate::query::invalidate_caches(root);
2510 crate::snapshot::remove_unreferenced(&cache_dir, &manifest, prev_manifest.as_ref());
2511
2512 *progress_status.lock().unwrap() = "Updating statistics...".to_string();
2514 if show_progress {
2515 pb.set_message("Updating statistics...".to_string());
2516 }
2517 self.cache.update_stats(&branch)?;
2519
2520 self.cache.update_schema_hash()?;
2522 self.cache.update_extraction_hash()?;
2523
2524 pb.finish_with_message("Indexing complete");
2525
2526 let mut stats = self.cache.stats_synced()?;
2529 stats.new_files = new_file_count;
2530 stats.modified_files = modified_file_count;
2531 stats.deleted_files = deleted_file_count;
2532 stats.unchanged_files = unchanged_file_count;
2533 stats.skipped_too_large = skipped_too_large;
2534 stats.skipped_bytes_too_large = skipped_bytes_too_large;
2535 stats.skipped_binary = skipped_binary;
2536 log::info!(
2537 "Indexing complete: {} files (new={}, modified={}, unchanged={})",
2538 stats.total_files,
2539 new_file_count,
2540 modified_file_count,
2541 unchanged_file_count
2542 );
2543
2544 Ok(stats)
2545 }
2546
2547 fn publish_delta(&self, c: DeltaChanges<'_>) -> Result<Option<IndexStats>> {
2554 use crate::snapshot::{IndexSnapshot, Manifest, SegmentFiles};
2555 use rusqlite::OptionalExtension;
2556 let cache_dir = self.cache.path().to_path_buf();
2557 let snapshot = IndexSnapshot::open(&cache_dir)?;
2558 let Some(prev) = snapshot.manifest().cloned() else {
2559 return Ok(None);
2560 };
2561 let base_len = snapshot.base_len();
2562 let delta_start = Instant::now();
2563 let mut laps = Laps::new();
2564
2565 let rels: Vec<String> = c.rewrites.iter().map(|r| r.rel.clone()).collect();
2566 let delta_len = snapshot.delta_len();
2567 let stable_end = base_len + delta_len;
2570
2571 let gone: std::collections::HashSet<String> = rels
2573 .iter()
2574 .cloned()
2575 .chain(c.deleted.iter().map(|(p, _)| p.clone()))
2576 .collect();
2577 let path_of = |id: u32| -> Option<&str> { snapshot.get_file_path(id)?.to_str() };
2578 let (mut new_dead_base, mut new_dead_delta) = (Vec::new(), Vec::new());
2579 for id in 0..stable_end {
2580 if path_of(id).is_some_and(|p| gone.contains(p)) {
2581 if id < base_len {
2582 new_dead_base.push(id);
2583 } else {
2584 new_dead_delta.push(id);
2585 }
2586 }
2587 }
2588 let kept_in = |range: std::ops::Range<u32>| -> Vec<(String, u32)> {
2589 range
2590 .filter_map(|id| {
2591 let p = path_of(id)?;
2592 (!gone.contains(p)).then(|| (p.to_string(), id))
2593 })
2594 .collect()
2595 };
2596 let delta_kept = kept_in(base_len..stable_end);
2597 let recent_kept = kept_in(stable_end..snapshot.id_bound());
2598
2599 let len_of = |kept: &[(String, u32)]| -> u64 {
2602 kept.iter()
2603 .map(|(_, id)| snapshot.file_len(*id).unwrap_or(0))
2604 .sum()
2605 };
2606 let rewrite_bytes: u64 = c.rewrites.iter().map(|r| r.size).sum();
2607 let (delta_kept_bytes, recent_kept_bytes) = (len_of(&delta_kept), len_of(&recent_kept));
2608 let (max_files, max_bytes) = self.merge_limits(prev.live_corpus_bytes);
2609 let all_files = c.rewrites.len() + delta_kept.len() + recent_kept.len();
2610 let all_bytes = rewrite_bytes + delta_kept_bytes + recent_kept_bytes;
2611 if all_files > max_files || all_bytes > max_bytes {
2612 log::info!(
2613 "Delta of {} files / ~{} bytes passes its limits ({} files / {} bytes)",
2614 all_files,
2615 all_bytes,
2616 max_files,
2617 max_bytes
2618 );
2619 return Ok(None);
2620 }
2621 let (recent_max_files, recent_max_bytes) = self.recent_limits(max_files, max_bytes);
2624 let fold = c.rewrites.len() + recent_kept.len() > recent_max_files
2625 || rewrite_bytes + recent_kept_bytes > recent_max_bytes;
2626
2627 let ctx = ProcessCtx {
2629 root: c.root,
2630 rels: &rels,
2631 stored: &c.stored,
2632 run_start: c.run_start,
2633 full_deps: false,
2634 tsconfigs: &c.resolver_configs.tsconfigs,
2635 };
2636 let mut read: Vec<Option<FileProcessingResult>> = c.pool.install(|| {
2637 (0..rels.len())
2638 .into_par_iter()
2639 .map_init(Vec::<u64>::new, |scratch, i| {
2640 self.process_file(&ctx, i, scratch)
2641 })
2642 .collect()
2643 });
2644 laps.lap("read");
2645
2646 let mut deleted = c.deleted;
2648 let mut live_files = c.live_files;
2649 for (k, r) in read.iter().enumerate() {
2650 if r.is_none() {
2651 live_files -= 1;
2652 if let Some(row) = c.stored.get(&rels[k]) {
2653 deleted.push((rels[k].clone(), row.id));
2654 }
2655 }
2656 }
2657
2658 enum Source {
2660 Fresh(usize),
2661 Old(u32),
2662 }
2663 let kept: Vec<(String, u32)> = if fold {
2664 delta_kept.into_iter().chain(recent_kept).collect()
2665 } else {
2666 recent_kept
2667 };
2668 let kept_seqs: HashMap<String, i64> = if kept.is_empty() {
2669 HashMap::new()
2670 } else {
2671 let conn = crate::cache::open_meta_db(cache_dir.join(crate::cache::META_DB))?;
2672 let mut stmt = conn.prepare("SELECT walk_seq FROM files WHERE path = ?")?;
2673 let mut out = HashMap::new();
2674 for (p, _) in &kept {
2675 if let Some(seq) = stmt.query_row([p], |r| r.get::<_, i64>(0)).optional()? {
2676 out.insert(p.clone(), seq);
2677 }
2678 }
2679 out
2680 };
2681 let mut entries: Vec<(i64, String, Source)> = Vec::new();
2682 for (k, r) in read.iter().enumerate() {
2683 if r.is_some() {
2684 entries.push((c.rewrites[k].walk_seq, rels[k].clone(), Source::Fresh(k)));
2685 }
2686 }
2687 for (p, id) in kept {
2688 let seq = kept_seqs.get(&p).copied().unwrap_or(i64::MAX);
2689 entries.push((seq, p, Source::Old(id)));
2690 }
2691 entries.sort_by_key(|(seq, _, _)| *seq);
2692 let mut segment_bytes = 0u64;
2693 for (_, _, src) in &entries {
2694 segment_bytes += match src {
2695 Source::Fresh(k) => read[*k].as_ref().map_or(0, |f| f.content.len() as u64),
2696 Source::Old(id) => snapshot.file_len(*id).unwrap_or(0),
2697 };
2698 }
2699 let (stable_files, stable_bytes) = if fold {
2700 (0, 0)
2701 } else {
2702 (
2703 snapshot.delta_len() as usize
2704 - new_dead_delta.len()
2705 - (0..delta_len)
2706 .filter(|&k| snapshot.is_dead(base_len + k))
2707 .count(),
2708 delta_kept_bytes,
2709 )
2710 };
2711 if entries.len() + stable_files > max_files || segment_bytes + stable_bytes > max_bytes {
2712 log::info!(
2713 "Delta of {} files / {} bytes passes its limits after reading",
2714 entries.len() + stable_files,
2715 segment_bytes + stable_bytes
2716 );
2717 return Ok(None);
2718 }
2719
2720 let mut tombstones: Vec<u32> = if fold {
2722 prev.tombstones
2724 .iter()
2725 .copied()
2726 .filter(|&id| id < base_len)
2727 .chain(new_dead_base.iter().copied())
2728 .collect()
2729 } else {
2730 prev.tombstones
2731 .iter()
2732 .copied()
2733 .chain(new_dead_base.iter().copied())
2734 .chain(new_dead_delta.iter().copied())
2735 .collect()
2736 };
2737 tombstones.sort_unstable();
2738 tombstones.dedup();
2739 let sizes_of = |ids: &[u32]| -> Vec<Vec<(crate::trigram::Trigram, u32)>> {
2740 c.pool.install(|| {
2741 ids.par_iter()
2742 .map_init(Vec::<u64>::new, |scratch, &id| {
2743 let content = snapshot.get_file_content(id).unwrap_or("");
2744 let run = crate::trigram_build::extract_trigram_run(content, scratch);
2745 crate::trigram_build::run_plan_sizes(&run)
2746 })
2747 .collect()
2748 })
2749 };
2750 let carried = |plan: Option<&crate::snapshot::TombPlan>| {
2751 plan.map(|t| t.entries().map(|(t, n)| (t, n as u64)).collect())
2752 .unwrap_or_default()
2753 };
2754 type TombMap = std::collections::BTreeMap<crate::trigram::Trigram, u64>;
2755 let mut touched_trigrams: Vec<crate::trigram::Trigram> = Vec::new();
2756 let mut tomb: TombMap = carried(snapshot.tomb());
2757 for sizes in sizes_of(&new_dead_base) {
2758 for (t, n) in sizes {
2759 *tomb.entry(t).or_default() += n as u64;
2760 touched_trigrams.push(t);
2761 }
2762 }
2763 let mut tomb_delta: TombMap = if fold {
2764 TombMap::new()
2765 } else {
2766 carried(snapshot.tomb_delta())
2767 };
2768 for sizes in sizes_of(&new_dead_delta) {
2769 for (t, n) in sizes {
2770 if !fold {
2771 *tomb_delta.entry(t).or_default() += n as u64;
2772 }
2773 touched_trigrams.push(t);
2774 }
2775 }
2776 laps.lap("tombstones");
2777
2778 let (delta_content, delta_trigrams, delta_plan, tomb_name) =
2780 crate::snapshot::delta_file_names(c.generation);
2781 let (recent_content, recent_trigrams, recent_plan, tomb_delta_name) =
2782 crate::snapshot::recent_file_names(c.generation);
2783 let (seg_content, seg_trigrams, seg_plan) = if fold {
2784 (delta_content, delta_trigrams, delta_plan)
2785 } else {
2786 (recent_content, recent_trigrams, recent_plan)
2787 };
2788 let segment = if entries.is_empty() {
2789 None
2790 } else {
2791 let runs: Vec<Option<TrigramRun>> = c.pool.install(|| {
2792 entries
2793 .par_iter()
2794 .map_init(Vec::<u64>::new, |scratch, (_, _, src)| match src {
2795 Source::Fresh(_) => None,
2796 Source::Old(id) => Some(crate::trigram_build::extract_trigram_run(
2797 snapshot.get_file_content(*id).unwrap_or(""),
2798 scratch,
2799 )),
2800 })
2801 .collect()
2802 });
2803 let mut builder = TrigramIndexBuilder::new(cache_dir.join("trigram_temp"));
2804 let mut writer = ContentWriter::new();
2805 let content_path = cache_dir.join(&seg_content);
2806 writer
2807 .init(content_path.clone())
2808 .context("Failed to initialize the delta content store")?;
2809 for ((_, rel, src), run) in entries.iter().zip(runs) {
2810 let path = PathBuf::from(rel);
2811 match src {
2812 Source::Fresh(k) => {
2813 let f = read[*k].as_mut().expect("read file");
2814 let run = std::mem::take(&mut f.trigram_run);
2815 builder.add_file(path.clone(), run);
2816 writer.add_file(path, &f.content);
2817 }
2818 Source::Old(id) => {
2819 builder.add_file(path.clone(), run.expect("kept file run"));
2820 writer.add_file(path, snapshot.get_file_content(*id)?);
2821 }
2822 }
2823 }
2824 let trigrams_path = cache_dir.join(&seg_trigrams);
2825 builder
2826 .write_with_plan(c.pool, &trigrams_path, Some(&cache_dir.join(&seg_plan)))
2827 .context("Failed to write the delta trigram index")?;
2828 writer
2829 .finalize_if_needed()
2830 .context("Failed to finalize the delta content store")?;
2831 Some(SegmentFiles {
2832 content: seg_content.clone(),
2833 trigrams: seg_trigrams.clone(),
2834 plan: Some(seg_plan.clone()),
2835 files: writer.file_count() as u64,
2836 content_bytes: std::fs::metadata(&content_path)?.len(),
2837 trigrams_bytes: std::fs::metadata(&trigrams_path)?.len(),
2838 })
2839 };
2840 let write_tomb = |map: &TombMap, name: String| -> Result<String> {
2841 let entries: Vec<(crate::trigram::Trigram, u32)> = map
2842 .iter()
2843 .map(|(&t, &n)| (t, n.min(u32::MAX as u64) as u32))
2844 .collect();
2845 crate::snapshot::write_tomb_file(&cache_dir.join(&name), &entries)?;
2846 Ok(name)
2847 };
2848 let tomb_file = if tombstones.iter().any(|&id| id < base_len) {
2849 Some(write_tomb(&tomb, tomb_name)?)
2850 } else {
2851 None
2852 };
2853 let tomb_delta_file = if tombstones.iter().any(|&id| id >= base_len) {
2854 Some(write_tomb(&tomb_delta, tomb_delta_name)?)
2855 } else {
2856 None
2857 };
2858 let (delta_files, recent_files) = if fold {
2859 (segment, None)
2860 } else {
2861 (prev.delta.clone(), segment)
2862 };
2863 laps.lap("write");
2864
2865 let segment_index =
2868 |files: &Option<SegmentFiles>| -> Result<Option<crate::trigram::TrigramIndex>> {
2869 let Some(f) = files else { return Ok(None) };
2870 let mut index = crate::trigram::TrigramIndex::load(cache_dir.join(&f.trigrams))?;
2871 if let Some(plan) = &f.plan {
2872 index.attach_plan(cache_dir.join(plan))?;
2873 }
2874 Ok(Some(index))
2875 };
2876 let new_segment = segment_index(if fold { &delta_files } else { &recent_files })?;
2877 if let Some(recent) = snapshot.recent() {
2878 touched_trigrams.extend(recent.trigrams.trigrams());
2879 }
2880 if fold && let Some(delta) = snapshot.delta() {
2881 touched_trigrams.extend(delta.trigrams.trigrams());
2882 }
2883 if let Some(index) = &new_segment {
2884 touched_trigrams.extend(index.trigrams());
2885 }
2886 touched_trigrams.sort_unstable();
2887 touched_trigrams.dedup();
2888 let plan_in = |index: Option<&crate::trigram::TrigramIndex>, t| -> i64 {
2889 index
2890 .and_then(|i| i.list_part(t))
2891 .map_or(0, |(_, plan)| plan as i64)
2892 };
2893 let old_delta = snapshot.delta().map(|d| &d.trigrams);
2894 let (new_delta, new_recent) = if fold {
2895 (new_segment.as_ref(), None)
2896 } else {
2897 (old_delta, new_segment.as_ref())
2898 };
2899 let mut live_trigrams = prev.live_trigrams as i64;
2900 for &t in &touched_trigrams {
2901 let was = snapshot.live_plan(t) > 0;
2902 let now = plan_in(Some(&snapshot.base().trigrams), t)
2903 + plan_in(new_delta, t)
2904 + plan_in(new_recent, t)
2905 - tomb.get(&t).copied().unwrap_or(0) as i64
2906 - tomb_delta.get(&t).copied().unwrap_or(0) as i64
2907 > 0;
2908 live_trigrams += now as i64 - was as i64;
2909 }
2910 let dead_set: std::collections::HashSet<u32> = tombstones.iter().copied().collect();
2911 let mut live_corpus = segment_bytes;
2914 let stable_bound = if fold { base_len } else { stable_end };
2915 for id in 0..stable_bound {
2916 if !dead_set.contains(&id) {
2917 live_corpus += snapshot.file_len(id).unwrap_or(0);
2918 }
2919 }
2920 laps.lap("live");
2921
2922 let manifest = Manifest {
2925 format: 0,
2926 generation: c.generation,
2927 base: prev.base.clone(),
2928 delta: delta_files,
2929 recent: recent_files,
2930 tombstones,
2931 tomb: tomb_file,
2932 tomb_delta: tomb_delta_file,
2933 live_trigrams: live_trigrams.max(0) as u64,
2934 publish_id: 0,
2935 live_corpus_bytes: live_corpus,
2936 checksum: String::new(),
2937 }
2938 .sealed();
2939 self.abort_point("delta-files");
2940 if !manifest.base_only() {
2941 crate::snapshot::unlink_fixed_names(&cache_dir);
2942 }
2943 self.abort_point("unlinked");
2944 crate::snapshot::write_manifest(&cache_dir, &manifest)?;
2945 crate::snapshot::link_fixed_names(&cache_dir, &manifest);
2946 self.abort_point("manifest");
2947 log::info!(
2948 "phase delta: {} delta + {} recent files ({} read{}), {} tombstones, {} ms",
2949 manifest.delta.as_ref().map_or(0, |d| d.files),
2950 manifest.recent.as_ref().map_or(0, |d| d.files),
2951 read.iter().filter(|r| r.is_some()).count(),
2952 if fold {
2953 ", folded into a new delta"
2954 } else {
2955 ""
2956 },
2957 manifest.tombstones.len(),
2958 delta_start.elapsed().as_millis()
2959 );
2960 laps.lap("manifest");
2961
2962 let meta_start = Instant::now();
2965 let now = chrono::Utc::now().timestamp();
2966 let flags = (c.dirty)();
2967 laps.lap("dirty");
2968 let mut rows: Vec<crate::cache::FileRow> = Vec::new();
2969 let mut row_k: Vec<usize> = Vec::new();
2970 for (k, r) in read.iter().enumerate() {
2971 if let Some(f) = r {
2972 rows.push(crate::cache::FileRow {
2973 path: rels[k].clone(),
2974 hash: f.hash.clone(),
2975 language: format!("{:?}", f.language),
2976 line_count: f.line_count,
2977 size: f.size,
2978 mtime_ns: f.mtime_ns,
2979 dirty: flags.rewrites.contains(&rels[k]),
2980 walk_seq: c.rewrites[k].walk_seq,
2981 });
2982 row_k.push(k);
2983 }
2984 }
2985 let paths_changed =
2986 !deleted.is_empty() || rows.iter().any(|r| !c.stored.contains_key(&r.path));
2987 let mut conn = crate::cache::open_meta_db(cache_dir.join(crate::cache::META_DB))?;
2988 let tx = conn
2989 .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)
2990 .context("Failed to begin meta.db transaction")?;
2991 let deleted_ids: Vec<i64> = deleted.iter().map(|(_, id)| *id).collect();
2992 crate::meta_update::delete_files(&tx, &deleted_ids)?;
2993 let ids = crate::meta_update::upsert_files(&tx, &rows, now)?;
2994 crate::meta_update::set_walk_seqs(&tx, &c.walk_moves)?;
2995 crate::meta_update::refresh_stats(&tx, &flags.touched)?;
2996 crate::meta_update::set_dirty_flags(&tx, &flags.flips)?;
2997 let branch_id = self
2998 .cache
2999 .get_or_create_branch_id(&tx, c.branch, c.commit)?;
3000 if c.sync_all_branch_rows {
3001 crate::meta_update::sync_branch_rows(&tx, branch_id, c.branch, now)?;
3002 } else {
3003 let branch_rows: Vec<(i64, &str)> = ids
3004 .iter()
3005 .zip(&rows)
3006 .map(|(id, row)| (*id, row.hash.as_str()))
3007 .collect();
3008 crate::meta_update::set_branch_rows(&tx, branch_id, &branch_rows, now)?;
3009 }
3010 log::info!(
3011 "meta.db: {} rows written, {} deleted, {} moved, {} touched, {} dirty flags",
3012 rows.len(),
3013 deleted_ids.len(),
3014 c.walk_moves.len(),
3015 flags.touched.len(),
3016 flags.flips.len()
3017 );
3018 laps.lap("rows");
3019
3020 let meta_path = cache_dir.join(crate::cache::META_DB);
3023 let cached = RESOLVER_CACHE
3024 .lock()
3025 .ok()
3026 .and_then(|mut slot| slot.take())
3027 .filter(|(path, publish, _)| {
3028 *path == meta_path && prev.publish_id != 0 && *publish == prev.publish_id
3029 })
3030 .map(|(_, _, resolver)| resolver);
3031 let resolver = match cached {
3032 Some(mut resolver) => {
3033 for (path, _) in &deleted {
3034 resolver.remove(path);
3035 }
3036 for (row, id) in rows.iter().zip(&ids) {
3037 resolver.insert(*id, &row.path);
3038 }
3039 resolver
3040 }
3041 None => crate::dependency::PathResolver::from_conn(&tx)
3042 .context("Failed to load file paths for dependency resolution")?,
3043 };
3044 laps.lap("resolver");
3045 let ctx = crate::dependency_resolve::ResolverContext::new(c.root, c.resolver_configs);
3046 let mut writer = crate::dependency::DependencyWriter::new(&tx);
3047 let mut resolved_here: std::collections::HashSet<i64> = std::collections::HashSet::new();
3048 for (&k, &file_id) in row_k.iter().zip(&ids) {
3049 let Some((imports, exports, declared)) =
3050 read[k].as_ref().and_then(|f| f.imports.as_ref())
3051 else {
3052 continue;
3053 };
3054 let rel = &rels[k];
3055 resolved_here.insert(file_id);
3056 let deps = ctx.resolve_file_imports(file_id, rel, imports.clone(), &resolver);
3057 writer.replace_dependencies(file_id, &deps)?;
3058 writer.replace_members(file_id, &ctx.package_members(rel, declared))?;
3059 writer.clear_exports(file_id)?;
3060 for export in exports {
3061 let resolved = ctx.resolve_export(rel, export, &resolver);
3062 writer.insert_export(
3063 file_id,
3064 export.exported_symbol.as_deref(),
3065 &export.source_path,
3066 resolved,
3067 export.line_number,
3068 )?;
3069 }
3070 }
3071 let (deps_written, exports_written) = writer.counts();
3072 let reresolved = if paths_changed {
3075 reresolve(&tx, &ctx, &resolver, &resolved_here)?
3076 } else {
3077 0
3078 };
3079 refresh_vendored(&tx, &c.resolver_configs.vendor, Some(&ids))?;
3080 laps.lap("deps");
3081
3082 crate::meta_update::set_statistic(
3083 &tx,
3084 RESOLVER_DIGEST_KEY,
3085 &c.resolver_configs.digest,
3086 now,
3087 )?;
3088 crate::meta_update::set_statistic(
3089 &tx,
3090 INDEX_GENERATION_KEY,
3091 &c.generation.to_string(),
3092 now,
3093 )?;
3094 CacheManager::update_branch_metadata_on(
3095 &tx,
3096 c.branch,
3097 c.commit,
3098 live_files,
3099 flags.git_dirty,
3100 )?;
3101 CacheManager::set_total_files_on(&tx, live_files, now)?;
3103 CacheManager::update_schema_hash_on(&tx)?;
3104 CacheManager::update_extraction_hash_on(&tx)?;
3105 laps.lap("stamps");
3106 tx.commit()?;
3107 self.abort_point("meta");
3108 if let Ok(mut slot) = RESOLVER_CACHE.lock() {
3109 *slot = Some((meta_path, manifest.publish_id, resolver));
3110 }
3111 laps.lap("commit");
3112 self.cache
3113 .checkpoint_wal()
3114 .context("Failed to checkpoint WAL")?;
3115 log::info!(
3116 "dependencies: {} rows, {} exports written; {} rows re-resolved",
3117 deps_written,
3118 exports_written,
3119 reresolved
3120 );
3121 log::info!(
3122 "phase meta.db (files, branches, dependencies, exports): {} ms",
3123 meta_start.elapsed().as_millis()
3124 );
3125
3126 crate::query::invalidate_caches(c.root);
3127 drop(snapshot);
3128 crate::snapshot::remove_unreferenced(&cache_dir, &manifest, Some(&prev));
3129 laps.lap("cleanup");
3130
3131 let (mut new_files, mut modified_files, mut unchanged_files) = c.breakdown_rest;
3132 for (k, r) in read.iter().enumerate() {
3133 if let Some(f) = r {
3134 match c.branch_hashes.get(&rels[k]) {
3135 None => new_files += 1,
3136 Some(old) if *old != f.hash => modified_files += 1,
3137 _ => unchanged_files += 1,
3138 }
3139 }
3140 }
3141 let mut stats = if c.full_stats {
3142 self.cache.stats_synced()?
3144 } else {
3145 self.light_stats(live_files, now)
3146 };
3147 laps.lap("stats");
3148 log::info!("delta update phases:{}", laps.line);
3149 stats.new_files = new_files;
3150 stats.modified_files = modified_files;
3151 stats.deleted_files = c.deleted_file_count;
3152 stats.unchanged_files = unchanged_files;
3153 stats.skipped_too_large = c.skipped.0;
3154 stats.skipped_bytes_too_large = c.skipped.1;
3155 stats.skipped_binary = c.skipped.2;
3156 Ok(Some(stats))
3157 }
3158
3159 fn reuse_file(
3164 &self,
3165 ctx: &ProcessCtx<'_>,
3166 snapshot: &crate::snapshot::IndexSnapshot,
3167 id: u32,
3168 i: usize,
3169 meta: &Option<crate::cache::FileStat>,
3170 trigram_scratch: &mut Vec<u64>,
3171 ) -> Option<FileProcessingResult> {
3172 let row = ctx.stored.get(&ctx.rels[i])?;
3173 let content = snapshot.get_file_content(id).ok()?.to_string();
3174 let file_path = &ctx.root.join(&ctx.rels[i]);
3175 let (size, mtime_ns) = match meta {
3176 Some(md) if !row.stat_matches(md) => (md.size(), md.recorded_mtime_ns(ctx.run_start)),
3177 _ => (row.size, row.mtime_ns),
3178 };
3179 let language = Language::from_path(file_path);
3180 let line_count = content.lines().count();
3181 let trigram_run = crate::trigram_build::extract_trigram_run(&content, trigram_scratch);
3182 let imports = ctx.full_deps.then(|| {
3183 let path_str = file_path.to_string_lossy().to_string();
3184 extract_imports(language, &content, &path_str, ctx.root, ctx.tsconfigs)
3185 });
3186 Some(FileProcessingResult {
3187 hash: row.hash.clone(),
3188 content,
3189 language,
3190 line_count,
3191 size,
3192 mtime_ns,
3193 imports,
3194 trigram_run,
3195 })
3196 }
3197
3198 fn process_file(
3201 &self,
3202 ctx: &ProcessCtx<'_>,
3203 i: usize,
3204 trigram_scratch: &mut Vec<u64>,
3205 ) -> Option<FileProcessingResult> {
3206 let file_path = &ctx.root.join(&ctx.rels[i]);
3207 let path_str = file_path.to_string_lossy().to_string();
3208
3209 let (size, mtime_ns) = std::fs::metadata(file_path)
3213 .map(|md| {
3214 (
3215 md.len(),
3216 crate::cache::recorded_mtime_ns(&md, ctx.run_start),
3217 )
3218 })
3219 .unwrap_or((0, 0));
3220
3221 let bytes = match std::fs::read(file_path) {
3227 Ok(b) => b,
3228 Err(e) => {
3229 log::warn!("Failed to read {}: {}", path_str, e);
3230 return None;
3231 }
3232 };
3233 let hash = self.hash_content(&bytes);
3234 let content = match String::from_utf8(bytes) {
3235 Ok(s) => s,
3236 Err(e) => String::from_utf8_lossy(e.as_bytes()).into_owned(),
3237 };
3238
3239 let language = Language::from_path(file_path);
3241
3242 let line_count = content.lines().count();
3244
3245 let trigram_run = crate::trigram_build::extract_trigram_run(&content, trigram_scratch);
3247
3248 let unchanged = ctx
3252 .stored
3253 .get(&ctx.rels[i])
3254 .is_some_and(|row| row.hash == hash);
3255 let imports = (ctx.full_deps || !unchanged)
3256 .then(|| extract_imports(language, &content, &path_str, ctx.root, ctx.tsconfigs));
3257
3258 Some(FileProcessingResult {
3259 hash,
3260 content,
3261 language,
3262 line_count,
3263 size,
3264 mtime_ns,
3265 imports,
3266 trigram_run,
3267 })
3268 }
3269
3270 fn run_marker(&self, cache_dir: &Path) -> std::time::SystemTime {
3273 let marker = cache_dir.join(RUN_MARKER);
3274 std::fs::write(&marker, b"")
3275 .and_then(|_| std::fs::metadata(&marker))
3276 .and_then(|md| md.modified())
3277 .unwrap_or_else(|_| std::time::SystemTime::now())
3278 }
3279
3280 fn stores_intact(&self, expected: usize, meta_generation: Option<u64>) -> bool {
3283 match crate::snapshot::check_published(self.cache.path()) {
3287 Ok(None) => {
3288 log::info!("Index written before the manifest - rebuilding");
3289 false
3290 }
3291 Ok(Some(manifest)) => {
3292 if Some(manifest.generation) != meta_generation {
3293 log::warn!(
3294 "Manifest generation {} but meta.db rows are generation {:?} (a run stopped between the two) - rebuilding",
3295 manifest.generation,
3296 meta_generation
3297 );
3298 false
3299 } else if manifest.live_files() as usize != expected {
3300 log::warn!(
3301 "Stores hold {} files but meta.db lists {} - rebuilding",
3302 manifest.live_files(),
3303 expected
3304 );
3305 false
3306 } else {
3307 true
3308 }
3309 }
3310 Err(e) => {
3311 log::warn!("Index stores unusable ({:#}) - rebuilding", e);
3312 false
3313 }
3314 }
3315 }
3316
3317 fn light_stats(&self, total_files: usize, now: i64) -> IndexStats {
3323 let (index_size_bytes, trigram_index_bytes, corpus_bytes) = self.cache.store_sizes();
3324 IndexStats {
3325 total_files,
3326 index_size_bytes,
3327 last_updated: chrono::DateTime::from_timestamp(now, 0)
3328 .unwrap_or_else(chrono::Utc::now)
3329 .to_rfc3339(),
3330 corpus_bytes,
3331 trigram_index_bytes,
3332 ..Default::default()
3333 }
3334 }
3335
3336 fn refresh_named(&self, r: RefreshNamed<'_>) -> Result<IndexStats> {
3340 let mut conn = crate::cache::open_meta_db(self.cache.path().join(crate::cache::META_DB))?;
3341 let tx = conn
3342 .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)
3343 .context("Failed to begin meta.db transaction")?;
3344 crate::meta_update::set_walk_seqs(&tx, r.walk_moves)?;
3345 crate::meta_update::refresh_stats(&tx, r.touched)?;
3346 crate::meta_update::set_dirty_flags(&tx, r.flips)?;
3347 if r.commit.is_some() {
3348 CacheManager::update_branch_metadata_on(
3349 &tx,
3350 r.branch,
3351 r.commit,
3352 r.live_files,
3353 r.git_dirty,
3354 )?;
3355 }
3356 CacheManager::update_stats_on(&tx, r.branch)?;
3357 tx.commit()?;
3358 let mut stats = self.light_stats(r.live_files, chrono::Utc::now().timestamp());
3359 stats.new_files = 0;
3360 stats.modified_files = 0;
3361 stats.deleted_files = 0;
3362 stats.unchanged_files = r.live_files;
3363 stats.skipped_too_large = r.skipped.0;
3364 stats.skipped_bytes_too_large = r.skipped.1;
3365 stats.skipped_binary = r.skipped.2;
3366 Ok(stats)
3367 }
3368
3369 fn refresh_unchanged(&self, r: RefreshUnchanged<'_>) -> Result<IndexStats> {
3370 let mut laps = Laps::new();
3371 let now = chrono::Utc::now().timestamp();
3372 let rows: Vec<&crate::meta_update::StoredFile> = r
3373 .rows
3374 .iter()
3375 .map(|row| row.expect("nothing was added: every file has a row"))
3376 .collect();
3377 let seqs = crate::meta_update::plan_walk_seq(
3378 &rows
3379 .iter()
3380 .map(|row| Some(row.walk_seq))
3381 .collect::<Vec<_>>(),
3382 );
3383 let mut walk = Vec::new();
3384 let mut flips = Vec::new();
3385 let mut touched = Vec::new();
3386 let no_dirty = r.dirty_paths.is_empty();
3387 for (i, rel) in r.rels.iter().enumerate() {
3388 let row = rows[i];
3389 let dirty = !no_dirty && r.dirty_paths.contains(rel);
3390 if seqs[i] != row.walk_seq {
3391 walk.push((row.id, seqs[i]));
3392 }
3393 if r.status[i] == FileStatus::Touched {
3394 let (size, mtime) = r.metas[i]
3395 .as_ref()
3396 .map(|md| (md.size(), md.recorded_mtime_ns(r.run_start)))
3397 .unwrap_or((0, 0));
3398 touched.push((row.id, size, mtime, dirty));
3399 } else if dirty != row.dirty {
3400 flips.push((row.id, dirty));
3401 }
3402 }
3403
3404 laps.lap("plan");
3405 let mut conn = crate::cache::open_meta_db(self.cache.path().join(crate::cache::META_DB))?;
3406 let tx = conn
3407 .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)
3408 .context("Failed to begin meta.db transaction")?;
3409 crate::meta_update::set_walk_seqs(&tx, &walk)?;
3410 crate::meta_update::refresh_stats(&tx, &touched)?;
3411 crate::meta_update::set_dirty_flags(&tx, &flips)?;
3412 let branch_id = self
3413 .cache
3414 .get_or_create_branch_id(&tx, r.branch, r.commit)?;
3415 laps.lap("rows");
3416 if !r.in_sync {
3417 crate::meta_update::sync_branch_rows(&tx, branch_id, r.branch, now)?;
3418 }
3419 laps.lap("branch_rows");
3420
3421 if let Some(git_dirty) = r.git_dirty {
3427 CacheManager::update_branch_metadata_on(
3428 &tx,
3429 r.branch,
3430 r.commit,
3431 r.rels.len(),
3432 git_dirty,
3433 )?;
3434 }
3435 CacheManager::set_total_files_on(&tx, r.rels.len(), now)?;
3437 tx.commit()?;
3438 laps.lap("commit");
3439
3440 let (new_files, modified_files, unchanged_files) = if r.in_sync {
3442 (0, 0, rows.len())
3443 } else {
3444 breakdown(
3445 r.rels
3446 .iter()
3447 .zip(&rows)
3448 .map(|(rel, row)| (rel.as_str(), row.hash.as_str())),
3449 r.existing_hashes,
3450 )
3451 };
3452 let mut files_by_language: HashMap<String, usize> = HashMap::new();
3455 let mut lines_by_language: HashMap<String, usize> = HashMap::new();
3456 for row in &rows {
3457 match files_by_language.get_mut(&row.language) {
3458 Some(n) => *n += 1,
3459 None => {
3460 files_by_language.insert(row.language.clone(), 1);
3461 }
3462 }
3463 match lines_by_language.get_mut(&row.language) {
3464 Some(n) => *n += row.line_count,
3465 None => {
3466 lines_by_language.insert(row.language.clone(), row.line_count);
3467 }
3468 }
3469 }
3470 let (index_size_bytes, trigram_index_bytes, corpus_bytes) = self.cache.store_sizes();
3471 let mut stats = IndexStats {
3472 total_files: r.rels.len(),
3473 index_size_bytes,
3474 last_updated: chrono::DateTime::from_timestamp(now, 0)
3475 .unwrap_or_else(chrono::Utc::now)
3476 .to_rfc3339(),
3477 files_by_language,
3478 lines_by_language,
3479 corpus_bytes,
3480 trigram_index_bytes,
3481 ..Default::default()
3482 };
3483 laps.lap("stats");
3484 log::info!("refresh phases:{}", laps.line);
3485 stats.new_files = new_files;
3486 stats.modified_files = modified_files;
3487 stats.unchanged_files = unchanged_files;
3488 stats.deleted_files = 0;
3489 stats.skipped_too_large = r.skipped.0;
3490 stats.skipped_bytes_too_large = r.skipped.1;
3491 stats.skipped_binary = r.skipped.2;
3492 Ok(stats)
3493 }
3494
3495 fn write_meta(&self, w: MetaWrite<'_>) -> Result<()> {
3501 let now = chrono::Utc::now().timestamp();
3502 let mut laps = Laps::new();
3503 let seqs = crate::meta_update::plan_walk_seq(
3504 &w.present
3505 .iter()
3506 .map(|&i| w.stored.get(&w.rels[i]).map(|s| s.walk_seq))
3507 .collect::<Vec<_>>(),
3508 );
3509 let written: HashMap<usize, &IndexedFile> =
3510 w.written.iter().map(|f| (f.index, f)).collect();
3511
3512 let mut rows: Vec<crate::cache::FileRow> = Vec::new();
3513 let mut row_files: Vec<usize> = Vec::new(); let mut walk = Vec::new();
3515 let mut flips = Vec::new();
3516 let mut touched = Vec::new();
3517 for (k, &i) in w.present.iter().enumerate() {
3518 let rel = &w.rels[i];
3519 let dirty = w.dirty_paths.contains(rel);
3520 let stored = w.stored.get(rel);
3521 match written.get(&i) {
3522 Some(f)
3523 if !(w.status[i] == FileStatus::Unchanged
3524 && stored.is_some_and(|row| row.hash == f.hash)) =>
3525 {
3526 rows.push(crate::cache::FileRow {
3527 path: rel.clone(),
3528 hash: f.hash.clone(),
3529 language: format!("{:?}", f.language),
3530 line_count: f.line_count,
3531 size: f.size,
3532 mtime_ns: f.mtime_ns,
3533 dirty,
3534 walk_seq: seqs[k],
3535 });
3536 row_files.push(i);
3537 }
3538 _ => {
3539 let Some(row) = stored else { continue };
3543 if seqs[k] != row.walk_seq {
3544 walk.push((row.id, seqs[k]));
3545 }
3546 if w.status[i] == FileStatus::Touched && !written.contains_key(&i) {
3547 let (size, mtime) = w.metas[i]
3548 .as_ref()
3549 .map(|md| (md.size(), md.recorded_mtime_ns(w.run_start)))
3550 .unwrap_or((0, 0));
3551 touched.push((row.id, size, mtime, dirty));
3552 } else if dirty != row.dirty {
3553 flips.push((row.id, dirty));
3554 }
3555 }
3556 }
3557 }
3558 let present: std::collections::HashSet<&str> =
3559 w.present.iter().map(|&i| w.rels[i].as_str()).collect();
3560 let deleted: Vec<i64> = w
3561 .stored
3562 .iter()
3563 .filter(|(p, _)| !present.contains(p.as_str()))
3564 .map(|(_, row)| row.id)
3565 .collect();
3566 let paths_changed = !deleted.is_empty()
3567 || w.present
3568 .iter()
3569 .any(|&i| !w.stored.contains_key(&w.rels[i]));
3570
3571 laps.lap("plan");
3572 let mut conn = crate::cache::open_meta_db(self.cache.path().join(crate::cache::META_DB))?;
3573 let tx = conn
3574 .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)
3575 .context("Failed to begin meta.db transaction")?;
3576
3577 crate::meta_update::delete_files(&tx, &deleted)?;
3578 let new_ids = crate::meta_update::upsert_files(&tx, &rows, now)?;
3579 crate::meta_update::set_walk_seqs(&tx, &walk)?;
3580 crate::meta_update::refresh_stats(&tx, &touched)?;
3581 crate::meta_update::set_dirty_flags(&tx, &flips)?;
3582 laps.lap("rows");
3583 let branch_id = self
3584 .cache
3585 .get_or_create_branch_id(&tx, w.branch, w.commit)?;
3586 crate::meta_update::sync_branch_rows(&tx, branch_id, w.branch, now)?;
3587 laps.lap("branch_rows");
3588 log::info!(
3589 "meta.db: {} rows written, {} deleted, {} moved, {} touched, {} dirty flags",
3590 rows.len(),
3591 deleted.len(),
3592 walk.len(),
3593 touched.len(),
3594 flips.len()
3595 );
3596
3597 let mut ids: HashMap<usize, i64> = w
3599 .written
3600 .iter()
3601 .filter_map(|f| w.stored.get(&w.rels[f.index]).map(|s| (f.index, s.id)))
3602 .collect();
3603 for (i, id) in row_files.iter().zip(new_ids) {
3604 ids.insert(*i, id);
3605 }
3606
3607 let resolver = crate::dependency::PathResolver::from_conn(&tx)
3608 .context("Failed to load file paths for dependency resolution")?;
3609 laps.lap("resolver");
3610 let ctx = crate::dependency_resolve::ResolverContext::new(w.root, w.resolver_configs);
3611 let mut writer = crate::dependency::DependencyWriter::new(&tx);
3612 if w.full_deps {
3613 writer.clear_all()?;
3614 }
3615 let mut resolved_here: std::collections::HashSet<i64> = std::collections::HashSet::new();
3617 for f in w.written {
3618 let Some((imports, exports, declared)) = &f.imports else {
3619 continue;
3620 };
3621 let rel = &w.rels[f.index];
3622 let file_id = ids[&f.index];
3623 resolved_here.insert(file_id);
3624 let deps = ctx.resolve_file_imports(file_id, rel, imports.clone(), &resolver);
3625 writer.replace_dependencies(file_id, &deps)?;
3626 writer.replace_members(file_id, &ctx.package_members(rel, declared))?;
3627 if !w.full_deps {
3628 writer.clear_exports(file_id)?;
3629 }
3630 for export in exports {
3631 let resolved = ctx.resolve_export(rel, export, &resolver);
3632 writer.insert_export(
3633 file_id,
3634 export.exported_symbol.as_deref(),
3635 &export.source_path,
3636 resolved,
3637 export.line_number,
3638 )?;
3639 }
3640 }
3641 let (deps_written, exports_written) = writer.counts();
3642 laps.lap("deps");
3643
3644 let mut reresolved = 0usize;
3647 if !w.full_deps && paths_changed {
3648 reresolved = reresolve(&tx, &ctx, &resolver, &resolved_here)?;
3649 }
3650 let written_ids: Vec<i64> = ids.values().copied().collect();
3651 refresh_vendored(
3652 &tx,
3653 &w.resolver_configs.vendor,
3654 (!w.full_deps).then_some(written_ids.as_slice()),
3655 )?;
3656
3657 crate::meta_update::set_statistic(
3658 &tx,
3659 RESOLVER_DIGEST_KEY,
3660 &w.resolver_configs.digest,
3661 now,
3662 )?;
3663 crate::meta_update::set_statistic(
3664 &tx,
3665 INDEX_GENERATION_KEY,
3666 &w.generation.to_string(),
3667 now,
3668 )?;
3669 laps.lap("reresolve");
3670 tx.commit()?;
3671 laps.lap("commit");
3672 log::info!("meta.db write phases:{}", laps.line);
3673 log::info!(
3674 "dependencies: {} rows, {} exports written; {} rows re-resolved",
3675 deps_written,
3676 exports_written,
3677 reresolved
3678 );
3679 Ok(())
3680 }
3681
3682 #[cfg(test)]
3688 fn discover_files(
3689 &self,
3690 root: &Path,
3691 stored: &HashMap<String, crate::meta_update::StoredFile>,
3692 ) -> Result<Discovered> {
3693 self.discover_files_in(root, stored, None)
3694 }
3695
3696 fn discover_files_in(
3700 &self,
3701 root: &Path,
3702 stored: &HashMap<String, crate::meta_update::StoredFile>,
3703 targets: Option<Arc<Targets>>,
3704 ) -> Result<Discovered> {
3705 let walked = self.walk_candidates(root, targets)?;
3706 Ok(Self::drop_binaries(root, walked, stored, None))
3707 }
3708
3709 fn walk_candidates(&self, root: &Path, targets: Option<Arc<Targets>>) -> Result<Walked> {
3713 let mut out = Walked::default();
3714
3715 let policy = self.path_policy(root);
3716 let walker = Self::targeted_walk_builder(root, &self.config, &policy, targets).build();
3717
3718 for entry in walker {
3719 let entry = entry?;
3720 let path = entry.path();
3721
3722 if !entry.file_type().map(|ft| ft.is_file()).unwrap_or(false) {
3724 continue;
3725 }
3726
3727 let Some(lang) = policy.classify(path) else {
3729 continue;
3730 };
3731
3732 let mut size = 0u64;
3734 let metadata = std::fs::metadata(path).ok();
3735 if let Some(metadata) = &metadata {
3736 size = metadata.len();
3737 if size > self.config.max_file_size as u64 {
3738 log::debug!("Skipping {} (too large: {} bytes)", path.display(), size);
3739 out.found.skipped_too_large += 1;
3740 out.found.skipped_bytes_too_large += size;
3741 continue;
3742 }
3743 }
3744
3745 out.found.rels.push(normalize_rel(root, path));
3746 out.found.sizes.push(size);
3747 out.found
3748 .metas
3749 .push(metadata.as_ref().map(crate::cache::FileStat::of));
3750 out.non_code.push(!lang.is_code());
3751 }
3752
3753 Ok(out)
3754 }
3755
3756 fn drop_binaries(
3763 root: &Path,
3764 walked: Walked,
3765 stored: &HashMap<String, crate::meta_update::StoredFile>,
3766 pool: Option<&rayon::ThreadPool>,
3767 ) -> Discovered {
3768 let Walked {
3769 found: mut out,
3770 non_code,
3771 } = walked;
3772 let to_sniff: Vec<usize> = (0..out.rels.len())
3773 .filter(|&i| {
3774 non_code[i]
3775 && !stored
3776 .get(&out.rels[i])
3777 .zip(out.metas[i].as_ref())
3778 .is_some_and(|(row, md)| row.stat_matches(md))
3779 })
3780 .collect();
3781 let sniff = |i: &usize| looks_binary(&root.join(&out.rels[*i]));
3782 let binary: Vec<bool> = match pool {
3783 Some(pool) => pool.install(|| to_sniff.par_iter().map(sniff).collect()),
3784 None => to_sniff.iter().map(sniff).collect(),
3785 };
3786 let dropped: std::collections::HashSet<usize> = to_sniff
3787 .iter()
3788 .zip(binary)
3789 .filter_map(|(&i, b)| b.then_some(i))
3790 .collect();
3791 if dropped.is_empty() {
3792 return out;
3793 }
3794 for &i in &dropped {
3795 log::debug!("Skipping {} (binary)", out.rels[i]);
3796 }
3797 out.skipped_binary += dropped.len();
3798 fn keep<T>(v: &mut Vec<T>, dropped: &std::collections::HashSet<usize>) {
3799 let mut i = 0;
3800 v.retain(|_| {
3801 let k = !dropped.contains(&i);
3802 i += 1;
3803 k
3804 });
3805 }
3806 keep(&mut out.rels, &dropped);
3807 keep(&mut out.sizes, &dropped);
3808 keep(&mut out.metas, &dropped);
3809 out
3810 }
3811
3812 fn targeted_walk_builder(
3818 root: &Path,
3819 config: &IndexConfig,
3820 policy: &PathPolicy,
3821 targets: Option<Arc<Targets>>,
3822 ) -> WalkBuilder {
3823 let mut builder = Self::walk_builder(root, config, policy);
3824 if let Some(targets) = targets {
3825 let hidden = policy.hidden();
3827 let root_buf = root.to_path_buf();
3828 builder.filter_entry(move |e| {
3829 if hidden {
3830 let name = e.file_name();
3831 if name == ".git" || name == crate::cache::CACHE_DIR {
3832 return false;
3833 }
3834 }
3835 let rel = normalize_rel(&root_buf, e.path());
3836 targets.admits(&rel)
3837 });
3838 }
3839 builder
3840 }
3841
3842 pub(crate) fn walked_among(
3846 root: &Path,
3847 config: &IndexConfig,
3848 policy: &PathPolicy,
3849 rels: &[String],
3850 ) -> std::collections::HashSet<String> {
3851 let targets = Arc::new(Targets::new(rels));
3852 let walker =
3853 Self::targeted_walk_builder(root, config, policy, Some(Arc::clone(&targets))).build();
3854 walker
3855 .flatten()
3856 .filter(|e| e.file_type().is_some_and(|t| t.is_file()))
3857 .map(|e| normalize_rel(root, e.path()))
3858 .filter(|rel| targets.exact.contains(rel))
3859 .collect()
3860 }
3861
3862 pub fn walk_builder(root: &Path, config: &IndexConfig, policy: &PathPolicy) -> WalkBuilder {
3863 let mut builder = WalkBuilder::new(root);
3868 builder
3869 .follow_links(config.follow_symlinks)
3870 .hidden(!policy.hidden())
3871 .git_ignore(true) .git_global(false) .git_exclude(false); if policy.hidden() {
3875 builder.filter_entry(|e| {
3879 let name = e.file_name();
3880 name != ".git" && name != crate::cache::CACHE_DIR
3881 });
3882 }
3883 if let Some(ov) = policy.overrides() {
3885 builder.overrides(ov.clone());
3886 }
3887 builder
3888 }
3889
3890 pub fn is_indexable_path(path: &Path) -> bool {
3901 Self::is_indexable_path_with(path, None)
3902 }
3903
3904 pub fn is_indexable_path_with(path: &Path, policy: Option<&PathPolicy>) -> bool {
3909 let default_policy;
3910 let policy = match policy {
3911 Some(p) => p,
3912 None => {
3913 default_policy = PathPolicy::default();
3914 &default_policy
3915 }
3916 };
3917 policy.hidden_ok(path) && policy.classify(path).is_some()
3921 }
3922
3923 fn should_index_lang(&self, path: &Path) -> bool {
3925 PathPolicy::from_config(Path::new("."), &self.config)
3926 .classify(path)
3927 .is_some()
3928 }
3929
3930 #[allow(dead_code)]
3932 fn should_index(&self, path: &Path) -> bool {
3933 if !self.should_index_lang(path) {
3934 return false;
3935 }
3936
3937 if let Ok(metadata) = std::fs::metadata(path)
3939 && metadata.len() > self.config.max_file_size as u64
3940 {
3941 log::debug!(
3942 "Skipping {} (too large: {} bytes)",
3943 path.display(),
3944 metadata.len()
3945 );
3946 return false;
3947 }
3948
3949 true
3953 }
3954
3955 fn hash_content(&self, content: &[u8]) -> String {
3957 let hash = blake3::hash(content);
3958 hash.to_hex().to_string()
3959 }
3960
3961 #[cfg_attr(not(unix), allow(unused_variables))]
3966 fn check_disk_space(&self, root: &Path) -> Result<()> {
3967 let cache_path = self.cache.path();
3969
3970 #[cfg(unix)]
3972 {
3973 let test_file = cache_path.join(".space_check");
3976 match std::fs::write(&test_file, b"test") {
3977 Ok(_) => {
3978 let _ = std::fs::remove_file(&test_file);
3979
3980 if let Ok(output) = std::process::Command::new("df")
3982 .arg("-k")
3983 .arg(cache_path.parent().unwrap_or(root))
3984 .output()
3985 && let Ok(df_output) = String::from_utf8(output.stdout)
3986 {
3987 if let Some(line) = df_output.lines().nth(1) {
3989 let parts: Vec<&str> = line.split_whitespace().collect();
3990 if parts.len() >= 4
3991 && let Ok(available_kb) = parts[3].parse::<u64>()
3992 {
3993 let available_mb = available_kb / 1024;
3994
3995 if available_mb < 100 {
3997 log::warn!(
3998 "Low disk space: only {}MB available. Indexing may fail.",
3999 available_mb
4000 );
4001 output::warn(&format!(
4002 "Low disk space ({}MB available). Consider freeing up space.",
4003 available_mb
4004 ));
4005 } else {
4006 log::debug!("Available disk space: {}MB", available_mb);
4007 }
4008 }
4009 }
4010 }
4011
4012 Ok(())
4013 }
4014 Err(e) if e.kind() == std::io::ErrorKind::PermissionDenied => {
4015 anyhow::bail!(
4016 "Permission denied writing to cache directory: {}. Check file permissions.",
4017 cache_path.display()
4018 )
4019 }
4020 Err(e) => {
4021 log::warn!(
4023 "Failed to write test file (possible disk space issue): {}",
4024 e
4025 );
4026 Err(e).context(
4027 "Failed to verify disk space - indexing may fail due to insufficient space",
4028 )
4029 }
4030 }
4031 }
4032
4033 #[cfg(not(unix))]
4034 {
4035 let test_file = cache_path.join(".space_check");
4037 match std::fs::write(&test_file, b"test") {
4038 Ok(_) => {
4039 let _ = std::fs::remove_file(&test_file);
4040 Ok(())
4041 }
4042 Err(e) if e.kind() == std::io::ErrorKind::PermissionDenied => {
4043 anyhow::bail!(
4044 "Permission denied writing to cache directory: {}. Check file permissions.",
4045 cache_path.display()
4046 )
4047 }
4048 Err(e) => {
4049 log::warn!(
4050 "Failed to write test file (possible disk space issue): {}",
4051 e
4052 );
4053 Err(e).context(
4054 "Failed to verify disk space - indexing may fail due to insufficient space",
4055 )
4056 }
4057 }
4058 }
4059 }
4060}
4061
4062#[cfg(test)]
4063mod tests {
4064 use super::*;
4065 use std::fs;
4066 use tempfile::TempDir;
4067
4068 #[test]
4069 fn test_indexer_creation() {
4070 let temp = TempDir::new().unwrap();
4071 let cache = CacheManager::new(temp.path());
4072 let config = IndexConfig::default();
4073 let indexer = Indexer::new(cache, config);
4074
4075 assert!(indexer.cache.path().ends_with(".reflex"));
4076 }
4077
4078 #[test]
4079 fn test_hash_content() {
4080 let temp = TempDir::new().unwrap();
4081 let cache = CacheManager::new(temp.path());
4082 let config = IndexConfig::default();
4083 let indexer = Indexer::new(cache, config);
4084
4085 let content1 = b"hello world";
4086 let content2 = b"hello world";
4087 let content3 = b"different content";
4088
4089 let hash1 = indexer.hash_content(content1);
4090 let hash2 = indexer.hash_content(content2);
4091 let hash3 = indexer.hash_content(content3);
4092
4093 assert_eq!(hash1, hash2);
4095
4096 assert_ne!(hash1, hash3);
4098
4099 assert_eq!(hash1.len(), 64); }
4102
4103 #[test]
4104 fn test_should_index_rust_file() {
4105 let temp = TempDir::new().unwrap();
4106 let cache = CacheManager::new(temp.path());
4107 let config = IndexConfig::default();
4108 let indexer = Indexer::new(cache, config);
4109
4110 let rust_file = temp.path().join("test.rs");
4112 fs::write(&rust_file, "fn main() {}").unwrap();
4113
4114 assert!(indexer.should_index(&rust_file));
4115 }
4116
4117 #[test]
4118 fn test_should_index_unsupported_extension() {
4119 let temp = TempDir::new().unwrap();
4120 let cache = CacheManager::new(temp.path());
4121 let config = IndexConfig::default();
4122 let indexer = Indexer::new(cache, config);
4123
4124 let unsupported_file = temp.path().join("test.xyz");
4127 fs::write(&unsupported_file, "mystery format").unwrap();
4128 assert!(indexer.should_index(&unsupported_file));
4129
4130 let allowlist = Indexer::new(
4132 CacheManager::new(temp.path()),
4133 IndexConfig {
4134 mode: IndexMode::Allowlist,
4135 ..Default::default()
4136 },
4137 );
4138 assert!(!allowlist.should_index(&unsupported_file));
4139 let binary_file = temp.path().join("logo.png");
4140 fs::write(&binary_file, "not really a png").unwrap();
4141 assert!(!allowlist.should_index(&binary_file));
4142 }
4143
4144 #[test]
4145 fn test_binary_sniff() {
4146 assert!(!is_binary(b"plain text\n"));
4147 assert!(is_binary(b"\x89PNG\r\n\x1a\n\0\0"));
4148 let mut late = vec![b'a'; 64 * 1024];
4150 late.push(0);
4151 assert!(is_binary(&late));
4152 }
4153
4154 #[test]
4155 fn test_should_index_text_tier() {
4156 let temp = TempDir::new().unwrap();
4157 let indexer = Indexer::new(CacheManager::new(temp.path()), IndexConfig::default());
4158
4159 for name in [
4160 "notes.md",
4161 "config.yaml",
4162 "data.json",
4163 "api.proto",
4164 "run.sh",
4165 ] {
4166 let path = temp.path().join(name);
4167 fs::write(&path, "realm_marker").unwrap();
4168 assert!(indexer.should_index(&path), "{name} should be indexed");
4169 }
4170
4171 let lock = temp.path().join("package-lock.json");
4174 fs::write(&lock, "{}").unwrap();
4175 assert!(indexer.should_index(&lock));
4176 let allowlist = Indexer::new(
4177 CacheManager::new(temp.path()),
4178 IndexConfig {
4179 mode: IndexMode::Allowlist,
4180 ..Default::default()
4181 },
4182 );
4183 assert!(!allowlist.should_index(&lock));
4184 }
4185
4186 #[test]
4187 fn test_text_tier_can_be_disabled() {
4188 let temp = TempDir::new().unwrap();
4189 let indexer = Indexer::new(
4190 CacheManager::new(temp.path()),
4191 IndexConfig {
4192 text_tier: false,
4193 ..Default::default()
4194 },
4195 );
4196
4197 let md = temp.path().join("notes.md");
4198 fs::write(&md, "text").unwrap();
4199 assert!(!indexer.should_index(&md));
4200
4201 let rs = temp.path().join("main.rs");
4202 fs::write(&rs, "fn main() {}").unwrap();
4203 assert!(indexer.should_index(&rs), "code is unaffected");
4204 }
4205
4206 #[test]
4207 fn test_should_index_no_extension() {
4208 let temp = TempDir::new().unwrap();
4209 let cache = CacheManager::new(temp.path());
4210 let config = IndexConfig::default();
4211 let indexer = Indexer::new(cache, config);
4212
4213 let makefile = temp.path().join("Makefile");
4216 fs::write(&makefile, "all:\n\techo hello").unwrap();
4217 assert!(indexer.should_index(&makefile));
4218
4219 let readme = temp.path().join("README");
4220 fs::write(&readme, "hello").unwrap();
4221 assert!(indexer.should_index(&readme));
4222
4223 let allowlist = Indexer::new(
4224 CacheManager::new(temp.path()),
4225 IndexConfig {
4226 mode: IndexMode::Allowlist,
4227 ..Default::default()
4228 },
4229 );
4230 assert!(allowlist.should_index(&makefile));
4231 assert!(!allowlist.should_index(&readme));
4232 }
4233
4234 #[test]
4235 fn test_should_index_size_limit() {
4236 let temp = TempDir::new().unwrap();
4237 let cache = CacheManager::new(temp.path());
4238
4239 let config = IndexConfig {
4241 max_file_size: 100,
4242 ..Default::default()
4243 };
4244
4245 let indexer = Indexer::new(cache, config);
4246
4247 let small_file = temp.path().join("small.rs");
4249 fs::write(&small_file, "fn main() {}").unwrap();
4250 assert!(indexer.should_index(&small_file));
4251
4252 let large_file = temp.path().join("large.rs");
4254 let large_content = "a".repeat(150);
4255 fs::write(&large_file, large_content).unwrap();
4256 assert!(!indexer.should_index(&large_file));
4257 }
4258
4259 #[test]
4260 fn test_discover_files_empty_dir() {
4261 let temp = TempDir::new().unwrap();
4262 let cache = CacheManager::new(temp.path());
4263 let config = IndexConfig::default();
4264 let indexer = Indexer::new(cache, config);
4265
4266 let files = indexer
4267 .discover_files(temp.path(), &HashMap::new())
4268 .unwrap()
4269 .rels
4270 .iter()
4271 .map(std::path::PathBuf::from)
4272 .collect::<Vec<_>>();
4273 assert_eq!(files.len(), 0);
4274 }
4275
4276 #[test]
4277 fn test_discover_files_single_file() {
4278 let temp = TempDir::new().unwrap();
4279 let cache = CacheManager::new(temp.path());
4280 let config = IndexConfig::default();
4281 let indexer = Indexer::new(cache, config);
4282
4283 let rust_file = temp.path().join("main.rs");
4285 fs::write(&rust_file, "fn main() {}").unwrap();
4286
4287 let files = indexer
4288 .discover_files(temp.path(), &HashMap::new())
4289 .unwrap()
4290 .rels
4291 .iter()
4292 .map(std::path::PathBuf::from)
4293 .collect::<Vec<_>>();
4294 assert_eq!(files.len(), 1);
4295 assert!(files[0].ends_with("main.rs"));
4296 }
4297
4298 #[test]
4299 fn test_discover_files_multiple_languages() {
4300 let temp = TempDir::new().unwrap();
4301 let cache = CacheManager::new(temp.path());
4302 let config = IndexConfig::default();
4303 let indexer = Indexer::new(cache, config);
4304
4305 fs::write(temp.path().join("main.rs"), "fn main() {}").unwrap();
4307 fs::write(temp.path().join("script.py"), "print('hello')").unwrap();
4308 fs::write(temp.path().join("app.js"), "console.log('hi')").unwrap();
4309 fs::write(temp.path().join("README.md"), "# Project").unwrap();
4311 fs::write(temp.path().join("mystery.xyz"), "?").unwrap();
4314 fs::write(temp.path().join("blob.bin"), b"\0\x01\x02").unwrap();
4315
4316 let found = indexer
4317 .discover_files(temp.path(), &HashMap::new())
4318 .unwrap();
4319 assert_eq!(found.rels.len(), 5, "3 code files, the markdown, the .xyz");
4320 assert_eq!(found.skipped_binary, 1);
4321 }
4322
4323 #[test]
4324 fn test_discover_files_subdirectories() {
4325 let temp = TempDir::new().unwrap();
4326 let cache = CacheManager::new(temp.path());
4327 let config = IndexConfig::default();
4328 let indexer = Indexer::new(cache, config);
4329
4330 let src_dir = temp.path().join("src");
4332 fs::create_dir(&src_dir).unwrap();
4333 fs::write(src_dir.join("main.rs"), "fn main() {}").unwrap();
4334 fs::write(src_dir.join("lib.rs"), "pub mod test {}").unwrap();
4335
4336 let tests_dir = temp.path().join("tests");
4337 fs::create_dir(&tests_dir).unwrap();
4338 fs::write(tests_dir.join("test.rs"), "#[test] fn test() {}").unwrap();
4339
4340 let files = indexer
4341 .discover_files(temp.path(), &HashMap::new())
4342 .unwrap()
4343 .rels
4344 .iter()
4345 .map(std::path::PathBuf::from)
4346 .collect::<Vec<_>>();
4347 assert_eq!(files.len(), 3);
4348 }
4349
4350 #[test]
4351 fn test_discover_files_respects_gitignore() {
4352 let temp = TempDir::new().unwrap();
4353
4354 std::process::Command::new("git")
4356 .arg("init")
4357 .current_dir(temp.path())
4358 .output()
4359 .expect("Failed to initialize git repo");
4360
4361 let cache = CacheManager::new(temp.path());
4362 let config = IndexConfig::default();
4363 let indexer = Indexer::new(cache, config);
4364
4365 fs::write(temp.path().join(".gitignore"), "ignored/\n").unwrap();
4368
4369 fs::write(temp.path().join("included.rs"), "fn main() {}").unwrap();
4371 fs::write(temp.path().join("also_included.py"), "print('hi')").unwrap();
4372
4373 let ignored_dir = temp.path().join("ignored");
4374 fs::create_dir(&ignored_dir).unwrap();
4375 fs::write(ignored_dir.join("excluded.rs"), "fn test() {}").unwrap();
4376
4377 let files = indexer
4378 .discover_files(temp.path(), &HashMap::new())
4379 .unwrap()
4380 .rels
4381 .iter()
4382 .map(std::path::PathBuf::from)
4383 .collect::<Vec<_>>();
4384
4385 assert!(
4387 files.iter().any(|f| f.ends_with("included.rs")),
4388 "Should find included.rs"
4389 );
4390 assert!(
4391 files.iter().any(|f| f.ends_with("also_included.py")),
4392 "Should find also_included.py"
4393 );
4394
4395 assert!(
4398 !files.iter().any(|f| {
4399 let path_str = f.to_string_lossy();
4400 path_str.contains("ignored") && f.ends_with("excluded.rs")
4401 }),
4402 "Should NOT find excluded.rs in ignored/ directory (gitignore pattern)"
4403 );
4404
4405 assert_eq!(
4408 files.len(),
4409 2,
4410 "Should find exactly 2 files (not including .gitignore or ignored/excluded.rs)"
4411 );
4412 }
4413
4414 #[test]
4415 fn test_index_empty_directory() {
4416 let temp = TempDir::new().unwrap();
4417 let cache = CacheManager::new(temp.path());
4418 let config = IndexConfig::default();
4419 let indexer = Indexer::new(cache, config);
4420
4421 let stats = indexer.index(temp.path(), false).unwrap();
4422
4423 assert_eq!(stats.total_files, 0);
4424 }
4425
4426 #[test]
4427 fn test_index_single_rust_file() {
4428 let temp = TempDir::new().unwrap();
4429 let project_root = temp.path().join("project");
4430 fs::create_dir(&project_root).unwrap();
4431
4432 let cache = CacheManager::new(&project_root);
4433 let config = IndexConfig::default();
4434 let indexer = Indexer::new(cache, config);
4435
4436 fs::write(
4438 project_root.join("main.rs"),
4439 "fn main() { println!(\"Hello\"); }",
4440 )
4441 .unwrap();
4442
4443 let stats = indexer.index(&project_root, false).unwrap();
4444
4445 assert_eq!(stats.total_files, 1);
4446 assert!(stats.files_by_language.contains_key("Rust"));
4447 }
4448
4449 #[test]
4450 fn test_index_multiple_files() {
4451 let temp = TempDir::new().unwrap();
4452 let project_root = temp.path().join("project");
4453 fs::create_dir(&project_root).unwrap();
4454
4455 let cache = CacheManager::new(&project_root);
4456 let config = IndexConfig::default();
4457 let indexer = Indexer::new(cache, config);
4458
4459 fs::write(project_root.join("main.rs"), "fn main() {}").unwrap();
4461 fs::write(project_root.join("lib.rs"), "pub fn test() {}").unwrap();
4462 fs::write(project_root.join("script.py"), "def main(): pass").unwrap();
4463
4464 let stats = indexer.index(&project_root, false).unwrap();
4465
4466 assert_eq!(stats.total_files, 3);
4467 assert_eq!(stats.files_by_language.get("Rust"), Some(&2));
4468 assert_eq!(stats.files_by_language.get("Python"), Some(&1));
4469 }
4470
4471 #[test]
4472 fn test_index_creates_trigram_index() {
4473 let temp = TempDir::new().unwrap();
4474 let project_root = temp.path().join("project");
4475 fs::create_dir(&project_root).unwrap();
4476
4477 let cache = CacheManager::new(&project_root);
4478 let config = IndexConfig::default();
4479 let indexer = Indexer::new(cache, config);
4480
4481 fs::write(project_root.join("main.rs"), "fn main() {}").unwrap();
4482
4483 indexer.index(&project_root, false).unwrap();
4484
4485 let trigrams_path = project_root.join(".reflex/trigrams.bin");
4487 assert!(trigrams_path.exists());
4488 }
4489
4490 #[test]
4491 fn test_index_creates_content_store() {
4492 let temp = TempDir::new().unwrap();
4493 let project_root = temp.path().join("project");
4494 fs::create_dir(&project_root).unwrap();
4495
4496 let cache = CacheManager::new(&project_root);
4497 let config = IndexConfig::default();
4498 let indexer = Indexer::new(cache, config);
4499
4500 fs::write(project_root.join("main.rs"), "fn main() {}").unwrap();
4501
4502 indexer.index(&project_root, false).unwrap();
4503
4504 let content_path = project_root.join(".reflex/content.bin");
4506 assert!(content_path.exists());
4507 }
4508
4509 #[test]
4510 fn test_index_incremental_no_changes() {
4511 let temp = TempDir::new().unwrap();
4512 let project_root = temp.path().join("project");
4513 fs::create_dir(&project_root).unwrap();
4514
4515 let cache = CacheManager::new(&project_root);
4516 let config = IndexConfig::default();
4517 let indexer = Indexer::new(cache, config);
4518
4519 fs::write(project_root.join("main.rs"), "fn main() {}").unwrap();
4520
4521 let stats1 = indexer.index(&project_root, false).unwrap();
4523 assert_eq!(stats1.total_files, 1);
4524
4525 let stats2 = indexer.index(&project_root, false).unwrap();
4527 assert_eq!(stats2.total_files, 1);
4528 }
4529
4530 #[test]
4531 fn test_index_incremental_with_changes() {
4532 let temp = TempDir::new().unwrap();
4533 let project_root = temp.path().join("project");
4534 fs::create_dir(&project_root).unwrap();
4535
4536 let cache = CacheManager::new(&project_root);
4537 let config = IndexConfig::default();
4538 let indexer = Indexer::new(cache, config);
4539
4540 let main_path = project_root.join("main.rs");
4541 fs::write(&main_path, "fn main() {}").unwrap();
4542
4543 indexer.index(&project_root, false).unwrap();
4545
4546 fs::write(&main_path, "fn main() { println!(\"changed\"); }").unwrap();
4548
4549 let stats = indexer.index(&project_root, false).unwrap();
4551 assert_eq!(stats.total_files, 1);
4552 }
4553
4554 #[test]
4555 fn test_index_incremental_new_file() {
4556 let temp = TempDir::new().unwrap();
4557 let project_root = temp.path().join("project");
4558 fs::create_dir(&project_root).unwrap();
4559
4560 let cache = CacheManager::new(&project_root);
4561 let config = IndexConfig::default();
4562 let indexer = Indexer::new(cache, config);
4563
4564 fs::write(project_root.join("main.rs"), "fn main() {}").unwrap();
4565
4566 let stats1 = indexer.index(&project_root, false).unwrap();
4568 assert_eq!(stats1.total_files, 1);
4569
4570 fs::write(project_root.join("lib.rs"), "pub fn test() {}").unwrap();
4572
4573 let stats2 = indexer.index(&project_root, false).unwrap();
4575 assert_eq!(stats2.total_files, 2);
4576 }
4577
4578 #[test]
4579 fn test_index_parallel_threads_config() {
4580 let temp = TempDir::new().unwrap();
4581 let project_root = temp.path().join("project");
4582 fs::create_dir(&project_root).unwrap();
4583
4584 let cache = CacheManager::new(&project_root);
4585
4586 let config = IndexConfig {
4588 parallel_threads: 2,
4589 ..Default::default()
4590 };
4591
4592 let indexer = Indexer::new(cache, config);
4593
4594 fs::write(project_root.join("main.rs"), "fn main() {}").unwrap();
4595
4596 let stats = indexer.index(&project_root, false).unwrap();
4597 assert_eq!(stats.total_files, 1);
4598 }
4599
4600 #[test]
4601 fn test_index_parallel_threads_auto() {
4602 let temp = TempDir::new().unwrap();
4603 let project_root = temp.path().join("project");
4604 fs::create_dir(&project_root).unwrap();
4605
4606 let cache = CacheManager::new(&project_root);
4607
4608 let config = IndexConfig {
4610 parallel_threads: 0,
4611 ..Default::default()
4612 };
4613
4614 let indexer = Indexer::new(cache, config);
4615
4616 fs::write(project_root.join("main.rs"), "fn main() {}").unwrap();
4617
4618 let stats = indexer.index(&project_root, false).unwrap();
4619 assert_eq!(stats.total_files, 1);
4620 }
4621
4622 #[test]
4623 fn test_index_respects_size_limit() {
4624 let temp = TempDir::new().unwrap();
4625 let project_root = temp.path().join("project");
4626 fs::create_dir(&project_root).unwrap();
4627
4628 let cache = CacheManager::new(&project_root);
4629
4630 let config = IndexConfig {
4632 max_file_size: 50,
4633 ..Default::default()
4634 };
4635
4636 let indexer = Indexer::new(cache, config);
4637
4638 fs::write(project_root.join("small.rs"), "fn a() {}").unwrap();
4640
4641 let large_content = "fn main() {}\n".repeat(10);
4643 fs::write(project_root.join("large.rs"), large_content).unwrap();
4644
4645 let stats = indexer.index(&project_root, false).unwrap();
4646
4647 assert_eq!(stats.total_files, 1);
4649 }
4650
4651 #[test]
4652 fn test_index_mixed_languages() {
4653 let temp = TempDir::new().unwrap();
4654 let project_root = temp.path().join("project");
4655 fs::create_dir(&project_root).unwrap();
4656
4657 let cache = CacheManager::new(&project_root);
4658 let config = IndexConfig::default();
4659 let indexer = Indexer::new(cache, config);
4660
4661 fs::write(project_root.join("main.rs"), "fn main() {}").unwrap();
4663 fs::write(project_root.join("test.py"), "def test(): pass").unwrap();
4664 fs::write(project_root.join("app.js"), "function main() {}").unwrap();
4665 fs::write(project_root.join("lib.go"), "func main() {}").unwrap();
4666
4667 let stats = indexer.index(&project_root, false).unwrap();
4668
4669 assert_eq!(stats.total_files, 4);
4670 assert!(stats.files_by_language.contains_key("Rust"));
4671 assert!(stats.files_by_language.contains_key("Python"));
4672 assert!(stats.files_by_language.contains_key("JavaScript"));
4673 assert!(stats.files_by_language.contains_key("Go"));
4674 }
4675
4676 #[test]
4677 fn test_index_updates_cache_stats() {
4678 let temp = TempDir::new().unwrap();
4679 let project_root = temp.path().join("project");
4680 fs::create_dir(&project_root).unwrap();
4681
4682 let cache = CacheManager::new(&project_root);
4683 let config = IndexConfig::default();
4684 let indexer = Indexer::new(cache, config);
4685
4686 fs::write(project_root.join("main.rs"), "fn main() {}").unwrap();
4687
4688 indexer.index(&project_root, false).unwrap();
4689
4690 let cache = CacheManager::new(&project_root);
4692 let stats = cache.stats().unwrap();
4693
4694 assert_eq!(stats.total_files, 1);
4695 assert!(stats.index_size_bytes > 0);
4696 }
4697}