use anyhow::{Context, Result};
use ignore::WalkBuilder;
use indicatif::{ProgressBar, ProgressDrawTarget, ProgressStyle};
use rayon::prelude::*;
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Instant;
use crate::cache::{CacheManager, WALK_SEQ_GAP};
use crate::content_store::ContentWriter;
use crate::models::{IndexConfig, IndexMode, IndexStats, Language};
#[cfg(unix)]
use crate::output;
use crate::parsers::c::CDependencyExtractor;
use crate::parsers::cpp::CppDependencyExtractor;
use crate::parsers::csharp::CSharpDependencyExtractor;
use crate::parsers::go::GoDependencyExtractor;
use crate::parsers::java::JavaDependencyExtractor;
use crate::parsers::kotlin::KotlinDependencyExtractor;
use crate::parsers::php::PhpDependencyExtractor;
use crate::parsers::python::PythonDependencyExtractor;
use crate::parsers::ruby::RubyDependencyExtractor;
use crate::parsers::rust::RustDependencyExtractor;
use crate::parsers::svelte::SvelteDependencyExtractor;
use crate::parsers::typescript::TypeScriptDependencyExtractor;
use crate::parsers::vue::VueDependencyExtractor;
use crate::parsers::zig::ZigDependencyExtractor;
use crate::parsers::{DependencyExtractor, ExportInfo, ImportInfo};
use crate::trigram_build::{TrigramIndexBuilder, TrigramRun};
pub type ProgressCallback = Arc<dyn Fn(usize, usize, String) + Send + Sync>;
struct FileProcessingResult {
hash: String,
content: String,
language: Language,
line_count: usize,
size: u64,
mtime_ns: i64,
imports: Option<Extracted>,
trigram_run: TrigramRun,
}
const RESOLVER_DIGEST_KEY: &str = "resolver_config_digest";
pub const INDEX_GENERATION_KEY: &str = "index_generation";
const RUN_MARKER: &str = ".index-run";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum FileStatus {
Unchanged,
Touched,
Modified,
Added,
}
struct IndexedFile {
index: usize,
hash: String,
language: Language,
line_count: usize,
size: u64,
mtime_ns: i64,
imports: Option<Extracted>,
}
struct Rewrite {
rel: String,
size: u64,
walk_seq: i64,
}
struct DirtyFlags {
rewrites: std::collections::HashSet<String>,
touched: Vec<(i64, u64, i64, bool)>,
flips: Vec<(i64, bool)>,
git_dirty: bool,
}
struct DeltaChanges<'a> {
root: &'a Path,
rewrites: Vec<Rewrite>,
deleted: Vec<(String, i64)>,
walk_moves: Vec<(i64, i64)>,
dirty: Box<dyn FnOnce() -> DirtyFlags + 'a>,
stored: HashMap<String, crate::meta_update::StoredFile>,
sync_all_branch_rows: bool,
full_stats: bool,
live_files: usize,
breakdown_rest: (usize, usize, usize),
branch_hashes: HashMap<String, String>,
branch: &'a str,
commit: Option<&'a str>,
resolver_configs: &'a crate::dependency_resolve::ResolverConfigs,
run_start: std::time::SystemTime,
generation: u64,
pool: &'a rayon::ThreadPool,
deleted_file_count: usize,
skipped: (usize, u64, usize),
}
struct ProcessCtx<'a> {
root: &'a Path,
rels: &'a [String],
stored: &'a HashMap<String, crate::meta_update::StoredFile>,
run_start: std::time::SystemTime,
full_deps: bool,
tsconfigs: &'a HashMap<PathBuf, crate::parsers::tsconfig::PathAliasMap>,
}
struct RefreshUnchanged<'a> {
rels: &'a [String],
metas: &'a [Option<crate::cache::FileStat>],
status: &'a [FileStatus],
rows: &'a [Option<&'a crate::meta_update::StoredFile>],
existing_hashes: &'a BranchHashes<'a>,
dirty_paths: &'a std::collections::HashSet<String>,
branch: &'a str,
in_sync: bool,
commit: Option<&'a str>,
git_dirty: Option<bool>,
run_start: std::time::SystemTime,
skipped: (usize, u64, usize),
}
struct RefreshNamed<'a> {
touched: &'a [(i64, u64, i64, bool)],
flips: &'a [(i64, bool)],
walk_moves: &'a [(i64, i64)],
branch: &'a str,
commit: Option<&'a str>,
git_dirty: bool,
live_files: usize,
skipped: (usize, u64, usize),
}
struct MetaWrite<'a> {
root: &'a Path,
rels: &'a [String],
present: &'a [usize],
written: &'a [IndexedFile],
metas: &'a [Option<crate::cache::FileStat>],
run_start: std::time::SystemTime,
stored: &'a HashMap<String, crate::meta_update::StoredFile>,
status: &'a [FileStatus],
dirty_paths: &'a std::collections::HashSet<String>,
branch: &'a str,
commit: Option<&'a str>,
resolver_configs: &'a crate::dependency_resolve::ResolverConfigs,
full_deps: bool,
generation: u64,
}
enum BranchHashes<'a> {
Synced(&'a HashMap<String, crate::meta_update::StoredFile>),
Loaded(HashMap<String, String>),
}
impl BranchHashes<'_> {
fn get(&self, path: &str) -> Option<&str> {
match self {
Self::Synced(stored) => stored.get(path).map(|row| row.hash.as_str()),
Self::Loaded(hashes) => hashes.get(path).map(String::as_str),
}
}
}
fn breakdown<'a>(
files: impl Iterator<Item = (&'a str, &'a str)>,
existing_hashes: &BranchHashes<'_>,
) -> (usize, usize, usize) {
let (mut new, mut modified, mut unchanged) = (0, 0, 0);
for (path, hash) in files {
match existing_hashes.get(path) {
None => new += 1,
Some(old) if old != hash => modified += 1,
_ => unchanged += 1,
}
}
(new, modified, unchanged)
}
fn refresh_vendored(
tx: &rusqlite::Connection,
rules: &crate::vendor::VendorRules,
ids: Option<&[i64]>,
) -> Result<usize> {
let mut changed: Vec<(i64, bool)> = Vec::new();
{
let mut visit = |id: i64, path: &str, old: bool| {
let new = rules.is_vendored(path, crate::models::Language::from_path(Path::new(path)));
if new != old {
changed.push((id, new));
}
};
match ids {
None => {
let mut stmt = tx.prepare("SELECT id, path, vendored FROM files")?;
let rows = stmt.query_map([], |r| {
Ok((
r.get::<_, i64>(0)?,
r.get::<_, String>(1)?,
r.get::<_, bool>(2)?,
))
})?;
for row in rows {
let (id, path, old) = row?;
visit(id, &path, old);
}
}
Some(ids) => {
let mut stmt = tx.prepare("SELECT path, vendored FROM files WHERE id = ?")?;
for &id in ids {
let (path, old): (String, bool) =
stmt.query_row([id], |r| Ok((r.get(0)?, r.get(1)?)))?;
visit(id, &path, old);
}
}
}
}
let mut stmt = tx.prepare("UPDATE files SET vendored = ? WHERE id = ?")?;
for (id, vendored) in &changed {
stmt.execute(rusqlite::params![vendored, id])?;
}
Ok(changed.len())
}
fn reresolve(
tx: &rusqlite::Connection,
ctx: &crate::dependency_resolve::ResolverContext<'_>,
resolver: &crate::dependency::PathResolver,
skip: &std::collections::HashSet<i64>,
) -> Result<usize> {
let mut changed: Vec<(i64, Option<i64>)> = Vec::new();
{
let mut stmt = tx.prepare(
"SELECT d.id, d.file_id, f.path, d.imported_path, d.resolved_file_id
FROM file_dependencies d JOIN files f ON f.id = d.file_id
WHERE d.import_type NOT IN ('external', 'stdlib')
AND d.resolved_package IS NULL",
)?;
let rows = stmt.query_map([], |r| {
Ok((
r.get::<_, i64>(0)?,
r.get::<_, i64>(1)?,
r.get::<_, String>(2)?,
r.get::<_, String>(3)?,
r.get::<_, Option<i64>>(4)?,
))
})?;
for row in rows {
let (id, file_id, path, imported_path, old) = row?;
if skip.contains(&file_id) {
continue;
}
let import = ImportInfo {
imported_path,
import_type: crate::models::ImportType::Internal,
line_number: 0,
imported_symbols: None,
};
let new = ctx.resolve_import(&path, &import, resolver);
if new != old {
changed.push((id, new));
}
}
}
let mut stmt = tx.prepare("UPDATE file_dependencies SET resolved_file_id = ? WHERE id = ?")?;
for (id, new) in &changed {
stmt.execute(rusqlite::params![new, id])?;
}
let mut total = changed.len();
let mut changed: Vec<(i64, Option<i64>)> = Vec::new();
{
let mut stmt = tx.prepare(
"SELECT e.id, e.file_id, f.path, e.source_path, e.resolved_source_id
FROM file_exports e JOIN files f ON f.id = e.file_id",
)?;
let rows = stmt.query_map([], |r| {
Ok((
r.get::<_, i64>(0)?,
r.get::<_, i64>(1)?,
r.get::<_, String>(2)?,
r.get::<_, String>(3)?,
r.get::<_, Option<i64>>(4)?,
))
})?;
for row in rows {
let (id, file_id, path, source_path, old) = row?;
if skip.contains(&file_id) {
continue;
}
let export = ExportInfo {
exported_symbol: None,
source_path,
line_number: 0,
};
let new = ctx.resolve_export(&path, &export, resolver);
if new != old {
changed.push((id, new));
}
}
}
let mut stmt = tx.prepare("UPDATE file_exports SET resolved_source_id = ? WHERE id = ?")?;
for (id, new) in &changed {
stmt.execute(rusqlite::params![new, id])?;
}
total += changed.len();
Ok(total)
}
type Extracted = (Vec<ImportInfo>, Vec<ExportInfo>, Vec<(String, String)>);
fn extract_imports(
language: Language,
content: &str,
path_str: &str,
root: &Path,
tsconfigs: &HashMap<PathBuf, crate::parsers::tsconfig::PathAliasMap>,
) -> Extracted {
let mut parsed_exports: Vec<ExportInfo> = Vec::new();
let mut declared_namespaces: Vec<(String, String)> = Vec::new();
let dependencies = match language {
Language::Rust => match RustDependencyExtractor::extract_dependencies(content) {
Ok(deps) => deps,
Err(e) => {
log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
Vec::new()
}
},
Language::Python => match PythonDependencyExtractor::extract_dependencies(content) {
Ok(deps) => deps,
Err(e) => {
log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
Vec::new()
}
},
Language::TypeScript | Language::JavaScript => {
let alias_map =
crate::dependency_resolve::find_nearest_tsconfig(path_str, root, tsconfigs);
match TypeScriptDependencyExtractor::extract_dependencies_and_exports(
content, alias_map,
) {
Ok((deps, exports)) => {
parsed_exports = exports;
deps
}
Err(e) => {
log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
Vec::new()
}
}
}
Language::Go => match GoDependencyExtractor::extract_dependencies(content) {
Ok(deps) => deps,
Err(e) => {
log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
Vec::new()
}
},
Language::Java => match JavaDependencyExtractor::extract_dependencies(content) {
Ok(deps) => deps,
Err(e) => {
log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
Vec::new()
}
},
Language::C => match CDependencyExtractor::extract_dependencies(content) {
Ok(deps) => deps,
Err(e) => {
log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
Vec::new()
}
},
Language::Cpp => match CppDependencyExtractor::extract_dependencies(content) {
Ok(deps) => deps,
Err(e) => {
log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
Vec::new()
}
},
Language::CSharp => {
match CSharpDependencyExtractor::extract_dependencies_and_members(content) {
Ok((deps, members)) => {
declared_namespaces = members;
deps
}
Err(e) => {
log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
Vec::new()
}
}
}
Language::PHP => match PhpDependencyExtractor::extract_dependencies(content) {
Ok(deps) => deps,
Err(e) => {
log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
Vec::new()
}
},
Language::Ruby => match RubyDependencyExtractor::extract_dependencies(content) {
Ok(deps) => deps,
Err(e) => {
log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
Vec::new()
}
},
Language::Kotlin => match KotlinDependencyExtractor::extract_dependencies(content) {
Ok(deps) => deps,
Err(e) => {
log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
Vec::new()
}
},
Language::Zig => match ZigDependencyExtractor::extract_dependencies(content) {
Ok(deps) => deps,
Err(e) => {
log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
Vec::new()
}
},
Language::Vue => {
let alias_map =
crate::dependency_resolve::find_nearest_tsconfig(path_str, root, tsconfigs);
match VueDependencyExtractor::extract_dependencies_and_exports(content, alias_map) {
Ok((deps, exports)) => {
parsed_exports = exports;
deps
}
Err(e) => {
log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
Vec::new()
}
}
}
Language::Svelte => match SvelteDependencyExtractor::extract_dependencies(content) {
Ok(deps) => deps,
Err(e) => {
log::warn!("Failed to extract dependencies from {}: {}", path_str, e);
Vec::new()
}
},
_ => Vec::new(),
};
let members = match language {
Language::Java | Language::Kotlin => {
crate::parsers::java::jvm_package_members(path_str, content)
}
Language::CSharp => declared_namespaces,
_ => Vec::new(),
};
(dependencies, parsed_exports, members)
}
pub struct Indexer {
cache: CacheManager,
config: IndexConfig,
batch_limits: Option<(usize, u64)>,
merge_limits: Option<(usize, u64)>,
recent_limits: Option<(usize, u64)>,
abort_at: Option<String>,
rules_pending: std::sync::Mutex<Option<String>>,
}
static RESOLVER_CACHE: Mutex<Option<(PathBuf, u64, crate::dependency::PathResolver)>> =
Mutex::new(None);
pub const RECENT_MAX_FILES: usize = 256;
pub const RECENT_BYTES_DIVISOR: u64 = 16;
#[doc(hidden)]
pub const ABORT_EXIT_CODE: i32 = 86;
pub const DELTA_MAX_FILES: usize = 2000;
pub const DELTA_MAX_CORPUS_PERCENT: u64 = 5;
pub const BATCH_MAX_FILES: usize = 5000;
pub const BATCH_MAX_BYTES: u64 = 48 << 20;
pub fn plan_batches(
sizes: &[u64],
max_files: usize,
max_bytes: u64,
) -> Vec<std::ops::Range<usize>> {
let max_files = max_files.max(1);
let mut batches = Vec::new();
let mut start = 0usize;
let mut bytes = 0u64;
for (i, &size) in sizes.iter().enumerate() {
let count = i - start;
if count > 0 && (count >= max_files || bytes.saturating_add(size) > max_bytes) {
batches.push(start..i);
start = i;
bytes = 0;
}
bytes = bytes.saturating_add(size);
}
if start < sizes.len() {
batches.push(start..sizes.len());
}
batches
}
#[derive(Clone, Debug)]
pub struct PathPolicy {
overrides: Option<ignore::overrides::Override>,
text_tier: bool,
languages: Vec<Language>,
mode: IndexMode,
hidden: bool,
}
impl Default for PathPolicy {
fn default() -> Self {
Self {
overrides: None,
text_tier: true,
languages: Vec::new(),
mode: IndexMode::Tracked,
hidden: false,
}
}
}
impl PathPolicy {
pub fn from_config(root: &Path, config: &IndexConfig) -> Self {
let mut policy = Self {
overrides: None,
text_tier: config.text_tier,
languages: config.languages.clone(),
mode: config.mode,
hidden: config.hidden,
};
if config.include_patterns.is_empty() && config.exclude_patterns.is_empty() {
return policy;
}
let mut builder = ignore::overrides::OverrideBuilder::new(root);
for pat in &config.include_patterns {
if let Err(e) = builder.add(pat) {
log::warn!("Invalid [index] include pattern '{}': {}", pat, e);
}
}
for pat in &config.exclude_patterns {
let negated = format!("!{}", pat.trim_start_matches('!'));
if let Err(e) = builder.add(&negated) {
log::warn!("Invalid [index] exclude pattern '{}': {}", pat, e);
}
}
match builder.build() {
Ok(ov) => policy.overrides = Some(ov),
Err(e) => log::warn!("Failed to build [index] include/exclude policy: {}", e),
}
policy
}
pub fn classify(&self, path: &Path) -> Option<Language> {
if !self.admits(path, false) {
return None;
}
let lang = Language::from_path(path);
let name = path.file_name().and_then(|n| n.to_str()).unwrap_or("");
match lang {
Language::Lock | Language::Generated => {
(self.mode == IndexMode::Tracked).then_some(lang)
}
Language::Text => {
if !self.text_tier {
return None;
}
match self.mode {
IndexMode::Tracked => Some(lang),
IndexMode::Allowlist => crate::models::is_text_tier_file(name).then_some(lang),
}
}
Language::Unknown => None,
code if !code.is_supported() => {
(self.mode == IndexMode::Tracked && self.text_tier).then_some(code)
}
code => {
if !self.languages.is_empty() && !self.languages.contains(&code) {
log::debug!(
"Skipping {} ({:?} not in configured languages)",
path.display(),
code
);
return None;
}
Some(code)
}
}
}
pub fn hidden_ok(&self, rel: &Path) -> bool {
rel.components()
.filter_map(|c| c.as_os_str().to_str())
.all(|seg| {
if seg == "." || seg == ".." {
return true;
}
if seg == ".git" || seg == crate::cache::CACHE_DIR {
return false;
}
self.hidden || !is_hidden_segment(seg)
})
}
pub fn hidden(&self) -> bool {
self.hidden
}
pub fn admits(&self, path: &Path, is_dir: bool) -> bool {
match &self.overrides {
None => true,
Some(ov) => !ov.matched(path, is_dir).is_ignore(),
}
}
pub fn overrides(&self) -> Option<&ignore::overrides::Override> {
self.overrides.as_ref()
}
}
#[derive(Debug, Default)]
struct Walked {
found: Discovered,
non_code: Vec<bool>,
}
#[derive(Debug, Default)]
struct Discovered {
rels: Vec<String>,
sizes: Vec<u64>,
metas: Vec<Option<crate::cache::FileStat>>,
skipped_too_large: usize,
skipped_bytes_too_large: u64,
skipped_binary: usize,
}
#[derive(Debug, Default)]
struct Targets {
exact: std::collections::HashSet<String>,
ancestors: std::collections::HashSet<String>,
}
impl Targets {
fn new(paths: &[String]) -> Self {
let mut out = Self::default();
for p in paths {
let mut end = p.len();
while let Some(slash) = p[..end].rfind('/') {
out.ancestors.insert(p[..slash].to_string());
end = slash;
}
out.exact.insert(p.clone());
}
out
}
fn covers(&self, rel: &str) -> bool {
if self.exact.contains(rel) {
return true;
}
let mut end = rel.len();
while let Some(slash) = rel[..end].rfind('/') {
if self.exact.contains(&rel[..slash]) {
return true;
}
end = slash;
}
false
}
fn admits(&self, rel: &str) -> bool {
rel.is_empty() || self.ancestors.contains(rel) || self.covers(rel)
}
}
fn place_rewrites(
conn: &rusqlite::Connection,
walk: &mut crate::walk_order::WalkOrder<'_>,
rewrites: &[(&str, Option<&crate::meta_update::StoredFile>)],
skip: &std::collections::HashSet<i64>,
) -> Result<Option<HashMap<String, i64>>> {
use crate::meta_update::row_near_seq;
use std::cmp::Ordering::{Greater, Less};
let mut skip = skip.clone();
let mut kept: HashMap<&str, i64> = HashMap::new();
let mut moved: std::collections::HashSet<&str> = std::collections::HashSet::new();
for _ in 0..4 {
kept.clear();
let mut changed = false;
for (path, row) in rewrites {
let Some(row) = row else { continue };
if moved.contains(path) {
continue;
}
let before = row_near_seq(conn, row.walk_seq, true, &skip)?;
let after = row_near_seq(conn, row.walk_seq + 1, false, &skip)?;
let fits = before
.as_ref()
.is_none_or(|(_, p, _)| walk.cmp(p, path) == Some(Less))
&& after
.as_ref()
.is_none_or(|(_, p, _)| walk.cmp(path, p) == Some(Less));
if fits {
kept.insert(path, row.walk_seq);
} else {
moved.insert(path);
skip.insert(row.id);
changed = true;
}
}
if !changed {
break;
}
}
let bounds: (Option<i64>, Option<i64>) =
conn.query_row("SELECT MIN(walk_seq), MAX(walk_seq) FROM files", [], |r| {
Ok((r.get(0)?, r.get(1)?))
})?;
let mut gaps: Vec<(&str, Option<i64>, Option<i64>)> = Vec::new();
for (path, row) in rewrites {
if row.is_some() && !moved.contains(path) {
continue;
}
let (before, after) = match bounds {
(Some(min), Some(max)) => {
let (mut lo, mut hi) = (min, max + 1);
while lo < hi {
let mid = lo + (hi - lo) / 2;
let after_path = match row_near_seq(conn, mid, false, &skip)? {
None => true,
Some((_, q, _)) => match walk.cmp(&q, path) {
Some(Greater) => true,
Some(Less) => false,
_ => return Ok(None),
},
};
if after_path {
hi = mid;
} else {
lo = mid + 1;
}
}
(
row_near_seq(conn, lo, true, &skip)?.map(|r| r.2),
row_near_seq(conn, lo, false, &skip)?.map(|r| r.2),
)
}
_ => (None, None),
};
gaps.push((path, before, after));
}
let mut out: HashMap<String, i64> = kept
.into_iter()
.map(|(p, seq)| (p.to_string(), seq))
.collect();
let mut i = 0;
while i < gaps.len() {
let (_, before, after) = gaps[i];
let mut j = i;
while j < gaps.len() && (gaps[j].1, gaps[j].2) == (before, after) {
j += 1;
}
let k = (j - i) as i64;
for (n, (path, _, _)) in gaps[i..j].iter().enumerate() {
let n = n as i64;
let seq = match (before, after) {
(Some(a), Some(b)) => {
let step = (b - a) / (k + 1);
if step < 1 {
return Ok(None);
}
a.checked_add(step * (n + 1))
}
(Some(a), None) => a.checked_add(WALK_SEQ_GAP * (n + 1)),
(None, Some(b)) => b.checked_sub(WALK_SEQ_GAP * (k - n)),
(None, None) => Some(WALK_SEQ_GAP * n),
};
let Some(seq) = seq else { return Ok(None) };
out.insert(path.to_string(), seq);
}
i = j;
}
Ok(Some(out))
}
fn normalize_rel(root: &Path, file_path: &Path) -> String {
match file_path.strip_prefix(root) {
Ok(rel) => rel.to_string_lossy().replace('\\', "/"),
Err(_) => file_path
.to_string_lossy()
.trim_start_matches("./")
.replace('\\', "/"),
}
}
pub fn is_hidden_segment(seg: &str) -> bool {
seg.len() > 1 && seg.starts_with('.') && seg != ".."
}
pub fn is_binary(bytes: &[u8]) -> bool {
memchr::memchr(0, bytes).is_some()
}
pub fn looks_binary(path: &Path) -> bool {
match std::fs::read(path) {
Ok(bytes) => is_binary(&bytes),
Err(_) => false,
}
}
struct Laps {
last: Instant,
line: String,
}
impl Laps {
fn new() -> Self {
Self {
last: Instant::now(),
line: String::new(),
}
}
fn lap(&mut self, name: &str) {
use std::fmt::Write;
let _ = write!(
self.line,
" {}={:.1}",
name,
self.last.elapsed().as_secs_f64() * 1000.0
);
self.last = Instant::now();
}
}
enum Named {
Skip,
Full,
Path(String),
}
fn classify_named(root: &Path, abs_root: &Path, path: &Path) -> Named {
let rel = if path.is_absolute() {
match path
.strip_prefix(root)
.or_else(|_| path.strip_prefix(abs_root))
{
Ok(rel) => rel,
Err(_) => return Named::Skip,
}
} else {
path
};
let mut parts: Vec<String> = Vec::new();
for component in rel.components() {
match component {
std::path::Component::Normal(s) => parts.push(s.to_string_lossy().into_owned()),
std::path::Component::CurDir => {}
_ => return Named::Full,
}
}
let Some(name) = parts.last() else {
return Named::Full;
};
if parts[0] == ".git" {
return Named::Skip;
}
if parts[0] == crate::cache::CACHE_DIR {
return if parts.len() == 2 && parts[1] == "config.toml" {
Named::Full
} else {
Named::Skip
};
}
if is_ignore_file_name(name) || crate::dependency_resolve::is_resolver_config_name(name) {
return Named::Full;
}
Named::Path(parts.join("/"))
}
pub(crate) const RULE_FILES_KEY: &str = "rule_files";
pub(crate) const FIXED_RULE_FILES: [&str; 4] =
[".reflex/config.toml", ".gitignore", ".ignore", ".rgignore"];
pub(crate) fn is_ignore_file_name(name: &str) -> bool {
matches!(name, ".gitignore" | ".ignore" | ".rgignore")
}
pub(crate) fn is_rule_file(rel: &str) -> bool {
rel == FIXED_RULE_FILES[0] || is_ignore_file_name(rel.rsplit('/').next().unwrap_or(rel))
}
pub(crate) fn rule_file_hash(root: &Path, rel: &str) -> String {
match std::fs::read(root.join(rel)) {
Ok(bytes) => blake3::hash(&bytes).to_hex().to_string(),
Err(_) => String::new(),
}
}
fn rule_files_record<'a>(root: &Path, dirty: impl IntoIterator<Item = &'a String>) -> String {
let mut record: std::collections::BTreeMap<String, String> = FIXED_RULE_FILES
.iter()
.map(|rel| (rel.to_string(), rule_file_hash(root, rel)))
.collect();
for rel in dirty {
if is_rule_file(rel) {
record.insert(rel.clone(), rule_file_hash(root, rel));
}
}
serde_json::to_string(&record).unwrap_or_default()
}
fn holds_resolver_config(dir: &Path) -> bool {
ignore::WalkBuilder::new(dir)
.standard_filters(false)
.build()
.flatten()
.any(|e| {
e.file_name()
.to_str()
.is_some_and(crate::dependency_resolve::is_resolver_config_name)
})
}
struct InvalidateOnDrop(std::path::PathBuf);
impl Drop for InvalidateOnDrop {
fn drop(&mut self) {
crate::query::invalidate_caches(&self.0);
}
}
impl Indexer {
pub fn new(cache: CacheManager, config: IndexConfig) -> Self {
Self {
cache,
config,
batch_limits: None,
merge_limits: None,
recent_limits: None,
abort_at: None,
rules_pending: std::sync::Mutex::new(None),
}
}
#[doc(hidden)]
pub fn set_batch_limits(&mut self, max_files: usize, max_bytes: u64) {
self.batch_limits = Some((max_files, max_bytes));
}
#[doc(hidden)]
pub fn set_merge_limits(&mut self, max_files: usize, max_bytes: u64) {
self.merge_limits = Some((max_files, max_bytes));
}
#[doc(hidden)]
pub fn set_abort_point(&mut self, point: &str) {
self.abort_at = Some(point.to_string());
}
fn abort_point(&self, point: &str) {
if self.abort_at.as_deref() == Some(point) {
std::process::exit(ABORT_EXIT_CODE);
}
}
#[doc(hidden)]
pub fn set_recent_limits(&mut self, max_files: usize, max_bytes: u64) {
self.recent_limits = Some((max_files, max_bytes));
}
fn recent_limits(&self, max_files: usize, max_bytes: u64) -> (usize, u64) {
self.recent_limits.unwrap_or((
RECENT_MAX_FILES.min(max_files),
max_bytes / RECENT_BYTES_DIVISOR,
))
}
fn merge_limits(&self, live_corpus_bytes: u64) -> (usize, u64) {
self.merge_limits.unwrap_or((
DELTA_MAX_FILES,
live_corpus_bytes * DELTA_MAX_CORPUS_PERCENT / 100,
))
}
fn batch_limits(&self) -> (usize, u64) {
if let Some(limits) = self.batch_limits {
return limits;
}
let files = std::env::var("REFLEX_INDEX_BATCH_FILES")
.ok()
.and_then(|v| v.parse::<usize>().ok())
.filter(|&n| n > 0)
.unwrap_or(BATCH_MAX_FILES);
let bytes = std::env::var("REFLEX_INDEX_BATCH_BYTES")
.ok()
.and_then(|v| v.parse::<u64>().ok())
.filter(|&n| n > 0)
.unwrap_or(BATCH_MAX_BYTES);
(files, bytes)
}
pub fn path_policy(&self, root: &Path) -> PathPolicy {
PathPolicy::from_config(root, &self.config)
}
pub fn index(&self, root: impl AsRef<Path>, show_progress: bool) -> Result<IndexStats> {
self.index_with_callback(root, show_progress, None)
}
pub fn update_paths(&self, root: impl AsRef<Path>, paths: &[PathBuf]) -> Result<IndexStats> {
let root = root.as_ref();
match self.try_update_paths(root, paths)? {
Some(stats) => Ok(stats),
None => {
log::info!("update_paths: the change needs a full index run");
self.index(root, false)
}
}
}
#[doc(hidden)]
pub fn try_update_paths(&self, root: &Path, paths: &[PathBuf]) -> Result<Option<IndexStats>> {
let started = Instant::now();
let mut laps = Laps::new();
let cache_dir = self.cache.path().to_path_buf();
if !cache_dir.join(crate::cache::META_DB).exists() {
return Ok(None);
}
let abs_root = root.canonicalize().unwrap_or_else(|_| root.to_path_buf());
let mut named: Vec<String> = Vec::new();
for path in paths {
match classify_named(root, &abs_root, path) {
Named::Skip => {}
Named::Full => return Ok(None),
Named::Path(rel) => named.push(rel),
}
}
named.sort();
named.dedup();
let in_git = crate::git::is_git_repo(root);
let spawn_status = |specs: Vec<String>| {
let root = root.to_path_buf();
std::thread::spawn(move || {
if specs.len() > 1000 {
crate::git::changed_paths(&root)
} else {
let refs: Vec<&str> = specs.iter().map(String::as_str).collect();
crate::git::changed_paths_in(&root, &refs)
}
})
};
let mut status_jobs = Vec::new();
if in_git && !named.is_empty() {
status_jobs.push(spawn_status(named.clone()));
}
let _index_lock = crate::atomic_write::IndexLock::acquire_with_timeout(
&cache_dir,
std::time::Duration::from_secs(self.config.lock_wait_secs),
)?;
crate::atomic_write::remove_stale_tmp(&cache_dir);
crate::query::invalidate_caches(root);
let _invalidate_on_exit = InvalidateOnDrop(root.to_path_buf());
self.yield_to_symbol_pass(&cache_dir)?;
laps.lap("lock");
self.cache.assert_writable(false)?;
if !self.cache.check_schema_hash().unwrap_or(false)
|| !self.cache.check_extraction_hash().unwrap_or(false)
{
return Ok(None);
}
let run_start = self.run_marker(&cache_dir);
laps.lap("checks");
let conn = crate::cache::open_meta_db(cache_dir.join(crate::cache::META_DB))?;
let file_count = crate::meta_update::count_files(&conn)?;
let stored_digest = crate::meta_update::get_statistic(&conn, RESOLVER_DIGEST_KEY)?;
let meta_generation = crate::meta_update::get_statistic(&conn, INDEX_GENERATION_KEY)?
.and_then(|g| g.parse::<u64>().ok());
let synced_branch =
crate::meta_update::get_statistic(&conn, crate::meta_update::SYNCED_BRANCH_KEY)?;
let (Some(generation), Ok(indexed)) =
(meta_generation, CacheManager::latest_branch_info_on(&conn))
else {
return Ok(None);
};
if !self.stores_intact(file_count, meta_generation) {
return Ok(None);
}
let generation = generation + 1;
laps.lap("snapshot");
let Some(resolver_configs) = crate::dependency_resolve::ResolverConfigs::from_saved_list(
root,
&cache_dir,
&self.config.vendored_patterns,
) else {
return Ok(None);
};
if stored_digest.as_deref() != Some(resolver_configs.digest.as_str()) {
return Ok(None);
}
laps.lap("configs");
let (branch, commit) = if in_git {
let Ok((commit, branch)) = crate::git::head_commit_and_branch(root) else {
return Ok(None);
};
(branch, Some(commit))
} else {
("_default".to_string(), None)
};
if synced_branch.as_deref() != Some(branch.as_str()) || branch != indexed.branch {
return Ok(None);
}
if let Some(commit) = &commit
&& *commit != indexed.commit_sha
{
let Ok(moved) = crate::git::diff_names(root, &indexed.commit_sha, commit) else {
return Ok(None);
};
let mut extra: Vec<String> = Vec::new();
for path in moved {
match classify_named(root, &abs_root, Path::new(&path)) {
Named::Skip => {}
Named::Full => return Ok(None),
Named::Path(rel) => {
if named.binary_search(&rel).is_err() {
extra.push(rel);
}
}
}
}
if !extra.is_empty() {
status_jobs.push(spawn_status(extra.clone()));
named.extend(extra);
named.sort();
named.dedup();
}
}
if resolver_configs.lists_any_under(root, &named)
|| named.iter().any(|rel| {
let dir = root.join(rel);
dir.is_dir() && holds_resolver_config(&dir)
})
{
return Ok(None);
}
let targets = Arc::new(Targets::new(&named));
laps.lap("git");
let stored = crate::meta_update::load_rows_under(&conn, &named)?;
let Discovered {
rels: found_rels,
metas: found_metas,
skipped_too_large,
skipped_bytes_too_large,
skipped_binary,
..
} = self.discover_files_in(root, &stored, Some(Arc::clone(&targets)))?;
let found: HashMap<&str, usize> = found_rels
.iter()
.enumerate()
.map(|(i, rel)| (rel.as_str(), i))
.collect();
laps.lap("walk");
let num_threads = crate::models::resolve_thread_count(self.config.parallel_threads, 32);
let pool = rayon::ThreadPoolBuilder::new()
.num_threads(num_threads)
.build()
.context("Failed to create thread pool")?;
let to_hash: Vec<usize> = (0..found_rels.len())
.filter(|&i| {
stored.get(&found_rels[i]).is_some_and(|row| {
!found_metas[i]
.as_ref()
.is_some_and(|md| row.stat_matches(md))
})
})
.collect();
let hashed: Vec<(usize, Option<String>)> = pool.install(|| {
to_hash
.par_iter()
.map(|&i| {
let hash = std::fs::read(root.join(&found_rels[i]))
.ok()
.map(|b| self.hash_content(&b));
(i, hash)
})
.collect()
});
let mut found_status: Vec<FileStatus> = found_rels
.iter()
.map(|rel| {
if stored.contains_key(rel) {
FileStatus::Unchanged
} else {
FileStatus::Added
}
})
.collect();
for (i, hash) in hashed {
found_status[i] = match hash {
Some(h) if h == stored[&found_rels[i]].hash => FileStatus::Touched,
_ => FileStatus::Modified,
};
}
let rewritten =
|i: usize| matches!(found_status[i], FileStatus::Added | FileStatus::Modified);
let deleted: Vec<&str> = {
let mut d: Vec<&str> = stored
.keys()
.map(String::as_str)
.filter(|p| !found.contains_key(p))
.collect();
d.sort_unstable();
d
};
laps.lap("hash");
let to_place: Vec<(&str, Option<&crate::meta_update::StoredFile>)> = (0..found_rels.len())
.filter(|&i| rewritten(i))
.map(|i| (found_rels[i].as_str(), stored.get(&found_rels[i])))
.collect();
let deleted_ids: std::collections::HashSet<i64> =
deleted.iter().map(|p| stored[*p].id).collect();
let mut walk_order = crate::walk_order::WalkOrder::new(root);
let Some(seq_of) = place_rewrites(&conn, &mut walk_order, &to_place, &deleted_ids)? else {
return Ok(None);
};
drop(conn);
let added = found_status
.iter()
.filter(|s| **s == FileStatus::Added)
.count();
let live_files = file_count - deleted.len() + added;
laps.lap("order");
let mut rewrites = Vec::new();
let mut rewrite_rows = HashMap::new();
for (i, rel) in found_rels.iter().enumerate() {
if rewritten(i) {
rewrites.push(Rewrite {
rel: rel.clone(),
size: found_metas[i].as_ref().map_or(0, |md| md.size()),
walk_seq: seq_of[rel],
});
if let Some(row) = stored.get(rel) {
rewrite_rows.insert(rel.clone(), row.clone());
}
}
}
let dirty_job = || -> DirtyFlags {
let mut now_dirty: std::collections::HashSet<String> = std::collections::HashSet::new();
let mut ok = true;
for job in status_jobs {
match job.join() {
Ok(Ok(paths)) => now_dirty.extend(paths),
_ => ok = false,
}
}
let is_dirty = |rel: &str| !ok || now_dirty.contains(rel);
let mut flags = DirtyFlags {
rewrites: std::collections::HashSet::new(),
touched: Vec::new(),
flips: Vec::new(),
git_dirty: in_git && (indexed.is_dirty || !ok || !now_dirty.is_empty()),
};
for (i, rel) in found_rels.iter().enumerate() {
let dirty = is_dirty(rel);
if rewritten(i) {
if dirty {
flags.rewrites.insert(rel.clone());
}
} else if let Some(row) = stored.get(rel) {
if found_status[i] == FileStatus::Touched {
let (size, mtime) = found_metas[i]
.as_ref()
.map(|md| (md.size(), md.recorded_mtime_ns(run_start)))
.unwrap_or((0, 0));
flags.touched.push((row.id, size, mtime, dirty));
} else if dirty != row.dirty {
flags.flips.push((row.id, dirty));
}
}
}
flags
};
let branch_hashes: HashMap<String, String> = {
let conn = crate::cache::open_meta_db(cache_dir.join(crate::cache::META_DB))?;
let mut stmt = conn.prepare(
"SELECT fb.hash FROM file_branches fb
JOIN files f ON f.id = fb.file_id
JOIN branches b ON b.id = fb.branch_id
WHERE b.name = ? AND f.path = ?",
)?;
let mut out = HashMap::new();
for r in &rewrites {
let hash: Option<String> = rusqlite::OptionalExtension::optional(
stmt.query_row(rusqlite::params![branch, r.rel], |row| row.get(0)),
)?;
if let Some(hash) = hash {
out.insert(r.rel.clone(), hash);
}
}
out
};
let skipped = (skipped_too_large, skipped_bytes_too_large, skipped_binary);
let rewritten_count = rewrites.len();
laps.lap("change_set");
let result = if deleted.is_empty() && rewrites.is_empty() {
let flags = dirty_job();
self.refresh_named(RefreshNamed {
touched: &flags.touched,
flips: &flags.flips,
walk_moves: &[],
branch: &branch,
commit: commit.as_deref(),
git_dirty: flags.git_dirty,
live_files,
skipped,
})
.map(Some)
} else {
let deleted_file_count = deleted.iter().filter(|p| !root.join(p).exists()).count();
self.publish_delta(DeltaChanges {
root,
rewrites,
deleted: deleted
.iter()
.map(|p| (p.to_string(), stored[*p].id))
.collect(),
walk_moves: Vec::new(),
dirty: Box::new(dirty_job),
stored: rewrite_rows,
sync_all_branch_rows: false,
full_stats: false,
live_files,
breakdown_rest: (0, 0, live_files - rewritten_count),
branch_hashes,
branch: &branch,
commit: commit.as_deref(),
resolver_configs: &resolver_configs,
run_start,
generation,
pool: &pool,
deleted_file_count,
skipped,
})
};
laps.lap("publish");
log::info!(
"update_paths: {} named paths, {} rewritten, {} deleted in {} ms;{}",
named.len(),
rewritten_count,
deleted.len(),
started.elapsed().as_millis(),
laps.line
);
result
}
const SYMBOL_PASS_YIELD_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);
fn yield_to_symbol_pass(&self, cache_dir: &Path) -> Result<()> {
use crate::background_indexer::BackgroundIndexer;
let Some(holder) = BackgroundIndexer::lock_holder(cache_dir) else {
BackgroundIndexer::clear_cancel(cache_dir);
return Ok(());
};
log::info!(
"Symbol indexing is running (pid {}); asking it to yield",
holder.pid
);
let _ = BackgroundIndexer::request_cancel(cache_dir);
let deadline = std::time::Instant::now() + Self::SYMBOL_PASS_YIELD_TIMEOUT;
let mut pause = std::time::Duration::from_millis(5);
while std::time::Instant::now() < deadline {
if !BackgroundIndexer::is_running(cache_dir) {
BackgroundIndexer::clear_cancel(cache_dir);
log::info!("Symbol indexing yielded; continuing");
return Ok(());
}
std::thread::sleep(pause);
pause = (pause * 2).min(std::time::Duration::from_millis(100));
}
BackgroundIndexer::clear_cancel(cache_dir);
let status = BackgroundIndexer::get_status(cache_dir).ok().flatten();
Err(crate::errors::ReflexError::SymbolIndexingInProgress {
pid: holder.pid,
started_at: holder.started_clock(),
processed: status.as_ref().map(|s| s.processed_files).unwrap_or(0),
total: status.as_ref().map(|s| s.total_files).unwrap_or(0),
}
.into())
}
pub fn index_with_callback(
&self,
root: impl AsRef<Path>,
show_progress: bool,
progress_callback: Option<ProgressCallback>,
) -> Result<IndexStats> {
let root = root.as_ref();
let stats = self.index_run(root, show_progress, progress_callback)?;
let pending = self
.rules_pending
.lock()
.unwrap_or_else(|e| e.into_inner())
.take();
if let Some(record) = pending {
let conn = crate::cache::open_meta_db(self.cache.path().join(crate::cache::META_DB))?;
crate::meta_update::set_statistic(
&conn,
RULE_FILES_KEY,
&record,
chrono::Utc::now().timestamp(),
)?;
crate::query::invalidate_caches(root);
}
Ok(stats)
}
fn index_run(
&self,
root: &Path,
show_progress: bool,
progress_callback: Option<ProgressCallback>,
) -> Result<IndexStats> {
log::info!("Indexing directory: {:?}", root);
let mut laps = Laps::new();
let cache_dir = self.cache.path().to_path_buf();
let _index_lock = crate::atomic_write::IndexLock::acquire_with_timeout(
&cache_dir,
std::time::Duration::from_secs(self.config.lock_wait_secs),
)?;
crate::atomic_write::remove_stale_tmp(&cache_dir);
crate::query::invalidate_caches(root);
let _invalidate_on_exit = InvalidateOnDrop(root.to_path_buf());
self.yield_to_symbol_pass(&cache_dir)?;
laps.lap("lock");
let num_threads = crate::models::resolve_thread_count(self.config.parallel_threads, 32);
log::info!(
"Using {} threads for parallel indexing (out of {} available)",
num_threads,
std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(4)
);
self.cache.assert_writable(false)?;
let schema_ok = self.cache.check_schema_hash().unwrap_or(false);
let extraction_ok = self.cache.check_extraction_hash().unwrap_or(false);
if schema_ok && extraction_ok {
self.cache.ensure_config()?;
} else {
self.cache.init()?;
}
if !schema_ok || !extraction_ok {
match self.cache.clear_symbol_cache() {
Ok(n) if n > 0 => {
log::info!("Cleared {} symbol cache rows written by other code", n)
}
Ok(_) => {}
Err(e) => log::warn!("Failed to clear the symbol cache: {}", e),
}
}
let run_start = self.run_marker(&cache_dir);
laps.lap("init");
let pool = rayon::ThreadPoolBuilder::new()
.num_threads(num_threads)
.build()
.context("Failed to create thread pool")?;
let meta_path = cache_dir.join(crate::cache::META_DB);
let phase_start = Instant::now();
let (git_state, resolver_configs, meta_read, walked) = std::thread::scope(|scope| {
let git = scope.spawn(|| {
let t = Instant::now();
let r = crate::git::get_git_state_optional(root);
(r, t.elapsed().as_millis())
});
let configs = scope.spawn(|| {
let t = Instant::now();
let r = crate::dependency_resolve::ResolverConfigs::discover(
root,
&self.config.vendored_patterns,
);
(r, t.elapsed().as_millis())
});
let disk = scope.spawn(|| self.check_disk_space(root));
let meta_read = scope.spawn(|| -> Result<_> {
let t = Instant::now();
let conn = crate::cache::open_meta_db(&meta_path)?;
let rows = crate::meta_update::load_stored_files(&conn)?;
log::debug!("meta.db rows read in {} ms", t.elapsed().as_millis());
Ok((
rows,
crate::meta_update::get_statistic(&conn, RESOLVER_DIGEST_KEY)?,
crate::meta_update::get_statistic(&conn, INDEX_GENERATION_KEY)?
.and_then(|g| g.parse::<u64>().ok()),
crate::meta_update::get_statistic(
&conn,
crate::meta_update::SYNCED_BRANCH_KEY,
)?,
crate::meta_update::get_statistic(&conn, RULE_FILES_KEY)?,
))
});
let t = Instant::now();
let walked = self.walk_candidates(root, None);
let walk_ms = t.elapsed().as_millis();
let (git, configs) = (git.join(), configs.join());
log::info!(
"discovery phase: walk {} ms, git {} ms, resolver configs {} ms",
walk_ms,
git.as_ref().map_or(0, |g| g.1),
configs.as_ref().map_or(0, |c| c.1)
);
let disk = disk
.join()
.unwrap_or_else(|_| Err(anyhow::anyhow!("disk space check panicked")));
(
git.map(|g| g.0),
configs.map(|c| c.0),
meta_read.join(),
walked.and_then(|w| disk.map(|()| w)),
)
});
let (stored, stored_digest, meta_generation, synced_branch, stored_rules) =
meta_read.map_err(|_| anyhow::anyhow!("meta.db read thread panicked"))??;
let discovered = walked.map(|w| Self::drop_binaries(root, w, &stored, Some(&pool)));
let prev_manifest = crate::snapshot::read_manifest(&cache_dir).ok().flatten();
let generation = prev_manifest
.as_ref()
.map(|m| m.generation)
.max(meta_generation)
.unwrap_or(0)
+ 1;
let git_state = git_state.map_err(|_| anyhow::anyhow!("git state thread panicked"))??;
let resolver_configs =
resolver_configs.map_err(|_| anyhow::anyhow!("resolver config walk panicked"))?;
if let Err(e) = resolver_configs.save_list(root, &cache_dir) {
log::warn!("Failed to save the resolver config list: {:#}", e);
}
let Discovered {
rels,
sizes,
metas,
skipped_too_large,
skipped_bytes_too_large,
skipped_binary,
} = discovered?;
let total_files = rels.len();
laps.lap("discover");
log::info!(
"Discovered {} files to index ({} skipped: too large, {} binary) in {} ms",
total_files,
skipped_too_large,
skipped_binary,
phase_start.elapsed().as_millis()
);
let branch = git_state
.as_ref()
.map(|s| s.branch.clone())
.unwrap_or_else(|| "_default".to_string());
if let Some(ref state) = git_state {
log::info!(
"Git state: branch='{}', commit='{}', dirty={}",
state.branch,
state.commit,
state.dirty
);
} else {
log::info!("Not a git repository, using default branch");
}
let commit = git_state.as_ref().map(|s| s.commit.clone());
let dirty_paths: std::collections::HashSet<String> = git_state
.as_ref()
.map(|s| s.dirty_paths.clone())
.unwrap_or_default();
let rules = rule_files_record(root, &dirty_paths);
*self.rules_pending.lock().unwrap_or_else(|e| e.into_inner()) =
(stored_rules.as_deref() != Some(rules.as_str())).then_some(rules);
let in_sync = synced_branch.as_deref() == Some(branch.as_str());
let existing_hashes = if in_sync {
BranchHashes::Synced(&stored)
} else {
BranchHashes::Loaded(self.cache.load_hashes_for_branch(&branch)?)
};
laps.lap("branch_hashes");
let classify_start = Instant::now();
let rows_of: Vec<Option<&crate::meta_update::StoredFile>> =
rels.iter().map(|rel| stored.get(rel)).collect();
let to_hash: Vec<usize> = (0..total_files)
.filter(|&i| {
rows_of[i]
.is_some_and(|row| !metas[i].as_ref().is_some_and(|md| row.stat_matches(md)))
})
.collect();
let hashed: Vec<(usize, Option<String>)> = pool.install(|| {
to_hash
.par_iter()
.map(|&i| {
(
i,
std::fs::read(root.join(&rels[i]))
.ok()
.map(|b| self.hash_content(&b)),
)
})
.collect()
});
let mut status: Vec<FileStatus> = rows_of
.iter()
.map(|row| {
if row.is_some() {
FileStatus::Unchanged
} else {
FileStatus::Added
}
})
.collect();
for (i, hash) in hashed {
status[i] = match (hash, rows_of[i]) {
(Some(h), Some(row)) if h == row.hash => FileStatus::Touched,
_ => FileStatus::Modified,
};
}
let walked_with_row = status.iter().filter(|s| **s != FileStatus::Added).count();
let gone: Vec<&String> = if walked_with_row == stored.len() {
Vec::new()
} else {
let discovered_set: std::collections::HashSet<&str> =
rels.iter().map(String::as_str).collect();
stored
.keys()
.filter(|p| !discovered_set.contains(p.as_str()))
.collect()
};
let deleted_file_count = gone.iter().filter(|p| !root.join(p).exists()).count();
let added = status.iter().filter(|s| **s == FileStatus::Added).count();
let modified = status
.iter()
.filter(|s| **s == FileStatus::Modified)
.count();
log::info!(
"phase classify: {} added, {} modified, {} touched, {} gone; {} files hashed in {} ms",
added,
modified,
status.iter().filter(|s| **s == FileStatus::Touched).count(),
gone.len(),
to_hash.len(),
classify_start.elapsed().as_millis()
);
let full_deps = !schema_ok
|| !extraction_ok
|| stored_digest.as_deref() != Some(resolver_configs.digest.as_str());
let content_changed = added > 0 || modified > 0 || !gone.is_empty();
laps.lap("classify");
let stores_ok = schema_ok && self.stores_intact(stored.len(), meta_generation);
laps.lap("snapshot");
if !content_changed && !full_deps && stores_ok {
log::info!("No files changed - skipping index rebuild");
log::info!("index run phases:{}", laps.line);
return self.refresh_unchanged(RefreshUnchanged {
rels: &rels,
metas: &metas,
status: &status,
rows: &rows_of,
existing_hashes: &existing_hashes,
dirty_paths: &dirty_paths,
branch: &branch,
in_sync,
commit: commit.as_deref(),
git_dirty: git_state.as_ref().map(|s| s.dirty),
run_start,
skipped: (skipped_too_large, skipped_bytes_too_large, skipped_binary),
});
}
if content_changed && !full_deps && stores_ok {
let seqs = crate::meta_update::plan_walk_seq(
&rels
.iter()
.map(|rel| stored.get(rel).map(|s| s.walk_seq))
.collect::<Vec<_>>(),
);
let mut rewrites = Vec::new();
let mut dirty_rewrites = std::collections::HashSet::new();
let mut rewrite_rows = HashMap::new();
let mut branch_hashes = HashMap::new();
let (mut touched, mut walk_moves, mut flips) = (Vec::new(), Vec::new(), Vec::new());
let mut rest: Vec<(&str, &str)> = Vec::new();
for i in 0..total_files {
let rel = &rels[i];
let dirty = dirty_paths.contains(rel);
match (status[i], stored.get(rel)) {
(FileStatus::Added | FileStatus::Modified, row) => {
rewrites.push(Rewrite {
rel: rel.clone(),
size: sizes[i],
walk_seq: seqs[i],
});
if dirty {
dirty_rewrites.insert(rel.clone());
}
if let Some(row) = row {
rewrite_rows.insert(rel.clone(), row.clone());
}
if let Some(h) = existing_hashes.get(rel) {
branch_hashes.insert(rel.clone(), h.to_string());
}
}
(_, Some(row)) => {
if seqs[i] != row.walk_seq {
walk_moves.push((row.id, seqs[i]));
}
if status[i] == FileStatus::Touched {
let (size, mtime) = metas[i]
.as_ref()
.map(|md| (md.size(), md.recorded_mtime_ns(run_start)))
.unwrap_or((0, 0));
touched.push((row.id, size, mtime, dirty));
} else if dirty != row.dirty {
flips.push((row.id, dirty));
}
rest.push((rel.as_str(), row.hash.as_str()));
}
(_, None) => unreachable!("an unchanged file has a row"),
}
}
let git_dirty = git_state.as_ref().map(|s| s.dirty).unwrap_or(false);
let changes = DeltaChanges {
root,
rewrites,
deleted: gone.iter().map(|p| ((*p).clone(), stored[*p].id)).collect(),
walk_moves,
dirty: Box::new(move || DirtyFlags {
rewrites: dirty_rewrites,
touched,
flips,
git_dirty,
}),
stored: rewrite_rows,
sync_all_branch_rows: !in_sync,
full_stats: true,
live_files: total_files,
breakdown_rest: breakdown(rest.into_iter(), &existing_hashes),
branch_hashes,
branch: &branch,
commit: commit.as_deref(),
resolver_configs: &resolver_configs,
run_start,
generation,
pool: &pool,
deleted_file_count,
skipped: (skipped_too_large, skipped_bytes_too_large, skipped_binary),
};
match self.publish_delta(changes)? {
Some(stats) => return Ok(stats),
None => log::info!("The delta would pass its limits - building a new base"),
}
}
if full_deps {
log::info!(
"Resolver configs, extraction code or schema changed - re-extracting every file's imports"
);
}
let mut files_indexed = 0;
let mut indexed: Vec<IndexedFile> = Vec::with_capacity(total_files);
let mut trigram_builder = TrigramIndexBuilder::new(self.cache.path().join("trigram_temp"));
let mut content_writer = ContentWriter::new();
let (content_name, trigrams_name, plan_name) = crate::snapshot::base_file_names(generation);
let content_path = cache_dir.join(&content_name);
content_writer
.init(content_path.clone())
.context("Failed to initialize content writer")?;
let pb = if show_progress {
let pb = ProgressBar::new(total_files as u64);
pb.set_draw_target(ProgressDrawTarget::stderr());
pb.set_style(
ProgressStyle::default_bar()
.template("[{elapsed_precise}] [{bar:40.cyan/blue}] {pos}/{len} files ({percent}%) {msg}")
.unwrap()
.progress_chars("=>-")
);
pb.enable_steady_tick(std::time::Duration::from_millis(100));
pb
} else {
ProgressBar::hidden()
};
let progress_counter = Arc::new(AtomicU64::new(0));
let progress_status = Arc::new(Mutex::new("Indexing files...".to_string()));
let batch_phase_start = Instant::now();
let mut pool_ms = 0u128;
let mut flush_ms = 0u128;
let merge_source: Option<(crate::snapshot::IndexSnapshot, HashMap<String, u32>)> =
if stores_ok {
crate::snapshot::IndexSnapshot::open(&cache_dir)
.ok()
.map(|snapshot| {
let ids = snapshot
.live_ids()
.filter_map(|id| {
Some((snapshot.get_file_path(id)?.to_str()?.to_string(), id))
})
.collect();
(snapshot, ids)
})
} else {
None
};
let from_snapshot = std::sync::atomic::AtomicUsize::new(0);
let counter_for_thread = Arc::clone(&progress_counter);
let status_for_thread = Arc::clone(&progress_status);
let pb_clone = pb.clone();
let callback_for_thread = progress_callback.clone();
let total_files_for_thread = total_files;
let progress_thread = if show_progress || callback_for_thread.is_some() {
Some(std::thread::spawn(move || {
loop {
let count = counter_for_thread.load(Ordering::Relaxed);
pb_clone.set_position(count);
if let Some(ref callback) = callback_for_thread {
let status = status_for_thread.lock().unwrap().clone();
callback(count as usize, total_files_for_thread, status);
}
if count >= total_files_for_thread as u64 {
break;
}
std::thread::sleep(std::time::Duration::from_millis(50));
}
}))
} else {
None
};
let (max_files, max_bytes) = self.batch_limits();
let batches = plan_batches(&sizes, max_files, max_bytes);
let num_batches = batches.len();
let spill_to_disk = num_batches > 1;
log::info!(
"Processing {} files in {} batches (<= {} files, ~{} MB each)",
total_files,
num_batches,
max_files,
max_bytes >> 20
);
for (batch_idx, batch_range) in batches.into_iter().enumerate() {
let batch_start = batch_range.start;
let batch_len = batch_range.len();
log::info!(
"Processing batch {}/{} ({} files)",
batch_idx + 1,
num_batches,
batch_len
);
let pool_start = Instant::now();
let counter_clone = Arc::clone(&progress_counter);
let tsconfigs = &resolver_configs.tsconfigs;
let ctx = ProcessCtx {
root,
rels: &rels,
stored: &stored,
run_start,
full_deps,
tsconfigs,
};
let results: Vec<Option<FileProcessingResult>> = pool.install(|| {
batch_range
.into_par_iter()
.map_init(Vec::<u64>::new, |trigram_scratch, i| {
let reused = match &merge_source {
Some((snapshot, ids))
if matches!(
status[i],
FileStatus::Unchanged | FileStatus::Touched
) =>
{
ids.get(&rels[i]).and_then(|&id| {
self.reuse_file(
&ctx,
snapshot,
id,
i,
&metas[i],
trigram_scratch,
)
})
}
_ => None,
};
let result = match reused {
Some(r) => {
from_snapshot.fetch_add(1, Ordering::Relaxed);
Some(r)
}
None => self.process_file(&ctx, i, trigram_scratch),
};
counter_clone.fetch_add(1, Ordering::Relaxed);
result
})
.collect()
});
pool_ms += pool_start.elapsed().as_millis();
if let Some((snapshot, _)) = &merge_source {
snapshot.release_pages();
}
for (offset, result) in results.into_iter().enumerate() {
let Some(result) = result else { continue };
let i = batch_start + offset;
let normalized_pathbuf = PathBuf::from(&rels[i]);
let _file_id =
trigram_builder.add_file(normalized_pathbuf.clone(), result.trigram_run);
content_writer.add_file(normalized_pathbuf, &result.content);
files_indexed += 1;
indexed.push(IndexedFile {
index: i,
hash: result.hash,
language: result.language,
line_count: result.line_count,
size: result.size,
mtime_ns: result.mtime_ns,
imports: result.imports,
});
}
let flush_msg = format!(
"Building trigram batch {}/{}...",
batch_idx + 1,
num_batches
);
if show_progress {
pb.set_message(flush_msg.clone());
}
*progress_status.lock().unwrap() = flush_msg;
let flush_start = Instant::now();
trigram_builder
.flush_batch(&pool, spill_to_disk)
.context("Failed to build trigram batch")?;
flush_ms += flush_start.elapsed().as_millis();
}
log::info!(
"phase read+extract: {} ms in pool, {} ms building trigram batches, {} ms total; {} files from the published stores",
pool_ms,
flush_ms,
batch_phase_start.elapsed().as_millis(),
from_snapshot.load(Ordering::Relaxed)
);
drop(merge_source);
if let Some(thread) = progress_thread {
let _ = thread.join();
}
if show_progress {
let final_count = progress_counter.load(Ordering::Relaxed);
pb.set_position(final_count);
}
*progress_status.lock().unwrap() = "Writing file metadata to database...".to_string();
if show_progress {
pb.set_message("Writing file metadata to database...".to_string());
}
let (new_file_count, modified_file_count, unchanged_file_count) = breakdown(
indexed
.iter()
.map(|f| (rels[f.index].as_str(), f.hash.as_str())),
&existing_hashes,
);
log::info!("Indexed {} files", files_indexed);
*progress_status.lock().unwrap() = "Writing trigram index...".to_string();
if show_progress {
pb.set_message("Writing trigram index...".to_string());
}
let trigrams_path = cache_dir.join(&trigrams_name);
let write_start = Instant::now();
trigram_builder
.write_with_plan(&pool, &trigrams_path, Some(&cache_dir.join(&plan_name)))
.context("Failed to write trigram index")?;
log::info!(
"phase trigram write: {} trigrams, {} files, {} ms",
trigram_builder.trigram_count(),
trigram_builder.file_count(),
write_start.elapsed().as_millis()
);
*progress_status.lock().unwrap() = "Finalizing content store...".to_string();
if show_progress {
pb.set_message("Finalizing content store...".to_string());
}
content_writer
.finalize_if_needed()
.context("Failed to finalize content store")?;
log::info!(
"Wrote {} files ({} bytes) to {}",
content_writer.file_count(),
content_writer.content_size(),
content_name
);
let manifest = crate::snapshot::Manifest::for_base(
generation,
crate::snapshot::SegmentFiles {
content: content_name.clone(),
trigrams: trigrams_name.clone(),
plan: Some(plan_name.clone()),
files: content_writer.file_count() as u64,
content_bytes: std::fs::metadata(&content_path)?.len(),
trigrams_bytes: std::fs::metadata(&trigrams_path)?.len(),
},
trigram_builder.trigram_count() as u64,
content_writer.content_size() as u64,
);
self.abort_point("base-files");
crate::snapshot::write_manifest(&cache_dir, &manifest)?;
crate::snapshot::link_fixed_names(&cache_dir, &manifest);
self.abort_point("base-manifest");
let meta_start = Instant::now();
let present: Vec<usize> = indexed.iter().map(|f| f.index).collect();
self.write_meta(MetaWrite {
root,
rels: &rels,
present: &present,
written: &indexed,
metas: &metas,
run_start,
stored: &stored,
status: &status,
dirty_paths: &dirty_paths,
branch: &branch,
commit: commit.as_deref(),
resolver_configs: &resolver_configs,
full_deps,
generation,
})?;
self.abort_point("base-meta");
self.cache.update_branch_metadata(
&branch,
commit.as_deref(),
indexed.len(),
git_state.as_ref().map(|s| s.dirty).unwrap_or(false),
)?;
self.cache
.checkpoint_wal()
.context("Failed to checkpoint WAL")?;
log::info!(
"phase meta.db (files, branches, dependencies, exports): {} ms",
meta_start.elapsed().as_millis()
);
crate::query::invalidate_caches(root);
crate::snapshot::remove_unreferenced(&cache_dir, &manifest, prev_manifest.as_ref());
*progress_status.lock().unwrap() = "Updating statistics...".to_string();
if show_progress {
pb.set_message("Updating statistics...".to_string());
}
self.cache.update_stats(&branch)?;
self.cache.update_schema_hash()?;
self.cache.update_extraction_hash()?;
pb.finish_with_message("Indexing complete");
let mut stats = self.cache.stats_synced()?;
stats.new_files = new_file_count;
stats.modified_files = modified_file_count;
stats.deleted_files = deleted_file_count;
stats.unchanged_files = unchanged_file_count;
stats.skipped_too_large = skipped_too_large;
stats.skipped_bytes_too_large = skipped_bytes_too_large;
stats.skipped_binary = skipped_binary;
log::info!(
"Indexing complete: {} files (new={}, modified={}, unchanged={})",
stats.total_files,
new_file_count,
modified_file_count,
unchanged_file_count
);
Ok(stats)
}
fn publish_delta(&self, c: DeltaChanges<'_>) -> Result<Option<IndexStats>> {
use crate::snapshot::{IndexSnapshot, Manifest, SegmentFiles};
use rusqlite::OptionalExtension;
let cache_dir = self.cache.path().to_path_buf();
let snapshot = IndexSnapshot::open(&cache_dir)?;
let Some(prev) = snapshot.manifest().cloned() else {
return Ok(None);
};
let base_len = snapshot.base_len();
let delta_start = Instant::now();
let mut laps = Laps::new();
let rels: Vec<String> = c.rewrites.iter().map(|r| r.rel.clone()).collect();
let delta_len = snapshot.delta_len();
let stable_end = base_len + delta_len;
let gone: std::collections::HashSet<String> = rels
.iter()
.cloned()
.chain(c.deleted.iter().map(|(p, _)| p.clone()))
.collect();
let path_of = |id: u32| -> Option<&str> { snapshot.get_file_path(id)?.to_str() };
let (mut new_dead_base, mut new_dead_delta) = (Vec::new(), Vec::new());
for id in 0..stable_end {
if path_of(id).is_some_and(|p| gone.contains(p)) {
if id < base_len {
new_dead_base.push(id);
} else {
new_dead_delta.push(id);
}
}
}
let kept_in = |range: std::ops::Range<u32>| -> Vec<(String, u32)> {
range
.filter_map(|id| {
let p = path_of(id)?;
(!gone.contains(p)).then(|| (p.to_string(), id))
})
.collect()
};
let delta_kept = kept_in(base_len..stable_end);
let recent_kept = kept_in(stable_end..snapshot.id_bound());
let len_of = |kept: &[(String, u32)]| -> u64 {
kept.iter()
.map(|(_, id)| snapshot.file_len(*id).unwrap_or(0))
.sum()
};
let rewrite_bytes: u64 = c.rewrites.iter().map(|r| r.size).sum();
let (delta_kept_bytes, recent_kept_bytes) = (len_of(&delta_kept), len_of(&recent_kept));
let (max_files, max_bytes) = self.merge_limits(prev.live_corpus_bytes);
let all_files = c.rewrites.len() + delta_kept.len() + recent_kept.len();
let all_bytes = rewrite_bytes + delta_kept_bytes + recent_kept_bytes;
if all_files > max_files || all_bytes > max_bytes {
log::info!(
"Delta of {} files / ~{} bytes passes its limits ({} files / {} bytes)",
all_files,
all_bytes,
max_files,
max_bytes
);
return Ok(None);
}
let (recent_max_files, recent_max_bytes) = self.recent_limits(max_files, max_bytes);
let fold = c.rewrites.len() + recent_kept.len() > recent_max_files
|| rewrite_bytes + recent_kept_bytes > recent_max_bytes;
let ctx = ProcessCtx {
root: c.root,
rels: &rels,
stored: &c.stored,
run_start: c.run_start,
full_deps: false,
tsconfigs: &c.resolver_configs.tsconfigs,
};
let mut read: Vec<Option<FileProcessingResult>> = c.pool.install(|| {
(0..rels.len())
.into_par_iter()
.map_init(Vec::<u64>::new, |scratch, i| {
self.process_file(&ctx, i, scratch)
})
.collect()
});
laps.lap("read");
let mut deleted = c.deleted;
let mut live_files = c.live_files;
for (k, r) in read.iter().enumerate() {
if r.is_none() {
live_files -= 1;
if let Some(row) = c.stored.get(&rels[k]) {
deleted.push((rels[k].clone(), row.id));
}
}
}
enum Source {
Fresh(usize),
Old(u32),
}
let kept: Vec<(String, u32)> = if fold {
delta_kept.into_iter().chain(recent_kept).collect()
} else {
recent_kept
};
let kept_seqs: HashMap<String, i64> = if kept.is_empty() {
HashMap::new()
} else {
let conn = crate::cache::open_meta_db(cache_dir.join(crate::cache::META_DB))?;
let mut stmt = conn.prepare("SELECT walk_seq FROM files WHERE path = ?")?;
let mut out = HashMap::new();
for (p, _) in &kept {
if let Some(seq) = stmt.query_row([p], |r| r.get::<_, i64>(0)).optional()? {
out.insert(p.clone(), seq);
}
}
out
};
let mut entries: Vec<(i64, String, Source)> = Vec::new();
for (k, r) in read.iter().enumerate() {
if r.is_some() {
entries.push((c.rewrites[k].walk_seq, rels[k].clone(), Source::Fresh(k)));
}
}
for (p, id) in kept {
let seq = kept_seqs.get(&p).copied().unwrap_or(i64::MAX);
entries.push((seq, p, Source::Old(id)));
}
entries.sort_by_key(|(seq, _, _)| *seq);
let mut segment_bytes = 0u64;
for (_, _, src) in &entries {
segment_bytes += match src {
Source::Fresh(k) => read[*k].as_ref().map_or(0, |f| f.content.len() as u64),
Source::Old(id) => snapshot.file_len(*id).unwrap_or(0),
};
}
let (stable_files, stable_bytes) = if fold {
(0, 0)
} else {
(
snapshot.delta_len() as usize
- new_dead_delta.len()
- (0..delta_len)
.filter(|&k| snapshot.is_dead(base_len + k))
.count(),
delta_kept_bytes,
)
};
if entries.len() + stable_files > max_files || segment_bytes + stable_bytes > max_bytes {
log::info!(
"Delta of {} files / {} bytes passes its limits after reading",
entries.len() + stable_files,
segment_bytes + stable_bytes
);
return Ok(None);
}
let mut tombstones: Vec<u32> = if fold {
prev.tombstones
.iter()
.copied()
.filter(|&id| id < base_len)
.chain(new_dead_base.iter().copied())
.collect()
} else {
prev.tombstones
.iter()
.copied()
.chain(new_dead_base.iter().copied())
.chain(new_dead_delta.iter().copied())
.collect()
};
tombstones.sort_unstable();
tombstones.dedup();
let sizes_of = |ids: &[u32]| -> Vec<Vec<(crate::trigram::Trigram, u32)>> {
c.pool.install(|| {
ids.par_iter()
.map_init(Vec::<u64>::new, |scratch, &id| {
let content = snapshot.get_file_content(id).unwrap_or("");
let run = crate::trigram_build::extract_trigram_run(content, scratch);
crate::trigram_build::run_plan_sizes(&run)
})
.collect()
})
};
let carried = |plan: Option<&crate::snapshot::TombPlan>| {
plan.map(|t| t.entries().map(|(t, n)| (t, n as u64)).collect())
.unwrap_or_default()
};
type TombMap = std::collections::BTreeMap<crate::trigram::Trigram, u64>;
let mut touched_trigrams: Vec<crate::trigram::Trigram> = Vec::new();
let mut tomb: TombMap = carried(snapshot.tomb());
for sizes in sizes_of(&new_dead_base) {
for (t, n) in sizes {
*tomb.entry(t).or_default() += n as u64;
touched_trigrams.push(t);
}
}
let mut tomb_delta: TombMap = if fold {
TombMap::new()
} else {
carried(snapshot.tomb_delta())
};
for sizes in sizes_of(&new_dead_delta) {
for (t, n) in sizes {
if !fold {
*tomb_delta.entry(t).or_default() += n as u64;
}
touched_trigrams.push(t);
}
}
laps.lap("tombstones");
let (delta_content, delta_trigrams, delta_plan, tomb_name) =
crate::snapshot::delta_file_names(c.generation);
let (recent_content, recent_trigrams, recent_plan, tomb_delta_name) =
crate::snapshot::recent_file_names(c.generation);
let (seg_content, seg_trigrams, seg_plan) = if fold {
(delta_content, delta_trigrams, delta_plan)
} else {
(recent_content, recent_trigrams, recent_plan)
};
let segment = if entries.is_empty() {
None
} else {
let runs: Vec<Option<TrigramRun>> = c.pool.install(|| {
entries
.par_iter()
.map_init(Vec::<u64>::new, |scratch, (_, _, src)| match src {
Source::Fresh(_) => None,
Source::Old(id) => Some(crate::trigram_build::extract_trigram_run(
snapshot.get_file_content(*id).unwrap_or(""),
scratch,
)),
})
.collect()
});
let mut builder = TrigramIndexBuilder::new(cache_dir.join("trigram_temp"));
let mut writer = ContentWriter::new();
let content_path = cache_dir.join(&seg_content);
writer
.init(content_path.clone())
.context("Failed to initialize the delta content store")?;
for ((_, rel, src), run) in entries.iter().zip(runs) {
let path = PathBuf::from(rel);
match src {
Source::Fresh(k) => {
let f = read[*k].as_mut().expect("read file");
let run = std::mem::take(&mut f.trigram_run);
builder.add_file(path.clone(), run);
writer.add_file(path, &f.content);
}
Source::Old(id) => {
builder.add_file(path.clone(), run.expect("kept file run"));
writer.add_file(path, snapshot.get_file_content(*id)?);
}
}
}
let trigrams_path = cache_dir.join(&seg_trigrams);
builder
.write_with_plan(c.pool, &trigrams_path, Some(&cache_dir.join(&seg_plan)))
.context("Failed to write the delta trigram index")?;
writer
.finalize_if_needed()
.context("Failed to finalize the delta content store")?;
Some(SegmentFiles {
content: seg_content.clone(),
trigrams: seg_trigrams.clone(),
plan: Some(seg_plan.clone()),
files: writer.file_count() as u64,
content_bytes: std::fs::metadata(&content_path)?.len(),
trigrams_bytes: std::fs::metadata(&trigrams_path)?.len(),
})
};
let write_tomb = |map: &TombMap, name: String| -> Result<String> {
let entries: Vec<(crate::trigram::Trigram, u32)> = map
.iter()
.map(|(&t, &n)| (t, n.min(u32::MAX as u64) as u32))
.collect();
crate::snapshot::write_tomb_file(&cache_dir.join(&name), &entries)?;
Ok(name)
};
let tomb_file = if tombstones.iter().any(|&id| id < base_len) {
Some(write_tomb(&tomb, tomb_name)?)
} else {
None
};
let tomb_delta_file = if tombstones.iter().any(|&id| id >= base_len) {
Some(write_tomb(&tomb_delta, tomb_delta_name)?)
} else {
None
};
let (delta_files, recent_files) = if fold {
(segment, None)
} else {
(prev.delta.clone(), segment)
};
laps.lap("write");
let segment_index =
|files: &Option<SegmentFiles>| -> Result<Option<crate::trigram::TrigramIndex>> {
let Some(f) = files else { return Ok(None) };
let mut index = crate::trigram::TrigramIndex::load(cache_dir.join(&f.trigrams))?;
if let Some(plan) = &f.plan {
index.attach_plan(cache_dir.join(plan))?;
}
Ok(Some(index))
};
let new_segment = segment_index(if fold { &delta_files } else { &recent_files })?;
if let Some(recent) = snapshot.recent() {
touched_trigrams.extend(recent.trigrams.trigrams());
}
if fold && let Some(delta) = snapshot.delta() {
touched_trigrams.extend(delta.trigrams.trigrams());
}
if let Some(index) = &new_segment {
touched_trigrams.extend(index.trigrams());
}
touched_trigrams.sort_unstable();
touched_trigrams.dedup();
let plan_in = |index: Option<&crate::trigram::TrigramIndex>, t| -> i64 {
index
.and_then(|i| i.list_part(t))
.map_or(0, |(_, plan)| plan as i64)
};
let old_delta = snapshot.delta().map(|d| &d.trigrams);
let (new_delta, new_recent) = if fold {
(new_segment.as_ref(), None)
} else {
(old_delta, new_segment.as_ref())
};
let mut live_trigrams = prev.live_trigrams as i64;
for &t in &touched_trigrams {
let was = snapshot.live_plan(t) > 0;
let now = plan_in(Some(&snapshot.base().trigrams), t)
+ plan_in(new_delta, t)
+ plan_in(new_recent, t)
- tomb.get(&t).copied().unwrap_or(0) as i64
- tomb_delta.get(&t).copied().unwrap_or(0) as i64
> 0;
live_trigrams += now as i64 - was as i64;
}
let dead_set: std::collections::HashSet<u32> = tombstones.iter().copied().collect();
let mut live_corpus = segment_bytes;
let stable_bound = if fold { base_len } else { stable_end };
for id in 0..stable_bound {
if !dead_set.contains(&id) {
live_corpus += snapshot.file_len(id).unwrap_or(0);
}
}
laps.lap("live");
let manifest = Manifest {
format: 0,
generation: c.generation,
base: prev.base.clone(),
delta: delta_files,
recent: recent_files,
tombstones,
tomb: tomb_file,
tomb_delta: tomb_delta_file,
live_trigrams: live_trigrams.max(0) as u64,
publish_id: 0,
live_corpus_bytes: live_corpus,
checksum: String::new(),
}
.sealed();
self.abort_point("delta-files");
if !manifest.base_only() {
crate::snapshot::unlink_fixed_names(&cache_dir);
}
self.abort_point("unlinked");
crate::snapshot::write_manifest(&cache_dir, &manifest)?;
crate::snapshot::link_fixed_names(&cache_dir, &manifest);
self.abort_point("manifest");
log::info!(
"phase delta: {} delta + {} recent files ({} read{}), {} tombstones, {} ms",
manifest.delta.as_ref().map_or(0, |d| d.files),
manifest.recent.as_ref().map_or(0, |d| d.files),
read.iter().filter(|r| r.is_some()).count(),
if fold {
", folded into a new delta"
} else {
""
},
manifest.tombstones.len(),
delta_start.elapsed().as_millis()
);
laps.lap("manifest");
let meta_start = Instant::now();
let now = chrono::Utc::now().timestamp();
let flags = (c.dirty)();
laps.lap("dirty");
let mut rows: Vec<crate::cache::FileRow> = Vec::new();
let mut row_k: Vec<usize> = Vec::new();
for (k, r) in read.iter().enumerate() {
if let Some(f) = r {
rows.push(crate::cache::FileRow {
path: rels[k].clone(),
hash: f.hash.clone(),
language: format!("{:?}", f.language),
line_count: f.line_count,
size: f.size,
mtime_ns: f.mtime_ns,
dirty: flags.rewrites.contains(&rels[k]),
walk_seq: c.rewrites[k].walk_seq,
});
row_k.push(k);
}
}
let paths_changed =
!deleted.is_empty() || rows.iter().any(|r| !c.stored.contains_key(&r.path));
let mut conn = crate::cache::open_meta_db(cache_dir.join(crate::cache::META_DB))?;
let tx = conn
.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)
.context("Failed to begin meta.db transaction")?;
let deleted_ids: Vec<i64> = deleted.iter().map(|(_, id)| *id).collect();
crate::meta_update::delete_files(&tx, &deleted_ids)?;
let ids = crate::meta_update::upsert_files(&tx, &rows, now)?;
crate::meta_update::set_walk_seqs(&tx, &c.walk_moves)?;
crate::meta_update::refresh_stats(&tx, &flags.touched)?;
crate::meta_update::set_dirty_flags(&tx, &flags.flips)?;
let branch_id = self
.cache
.get_or_create_branch_id(&tx, c.branch, c.commit)?;
if c.sync_all_branch_rows {
crate::meta_update::sync_branch_rows(&tx, branch_id, c.branch, now)?;
} else {
let branch_rows: Vec<(i64, &str)> = ids
.iter()
.zip(&rows)
.map(|(id, row)| (*id, row.hash.as_str()))
.collect();
crate::meta_update::set_branch_rows(&tx, branch_id, &branch_rows, now)?;
}
log::info!(
"meta.db: {} rows written, {} deleted, {} moved, {} touched, {} dirty flags",
rows.len(),
deleted_ids.len(),
c.walk_moves.len(),
flags.touched.len(),
flags.flips.len()
);
laps.lap("rows");
let meta_path = cache_dir.join(crate::cache::META_DB);
let cached = RESOLVER_CACHE
.lock()
.ok()
.and_then(|mut slot| slot.take())
.filter(|(path, publish, _)| {
*path == meta_path && prev.publish_id != 0 && *publish == prev.publish_id
})
.map(|(_, _, resolver)| resolver);
let resolver = match cached {
Some(mut resolver) => {
for (path, _) in &deleted {
resolver.remove(path);
}
for (row, id) in rows.iter().zip(&ids) {
resolver.insert(*id, &row.path);
}
resolver
}
None => crate::dependency::PathResolver::from_conn(&tx)
.context("Failed to load file paths for dependency resolution")?,
};
laps.lap("resolver");
let ctx = crate::dependency_resolve::ResolverContext::new(c.root, c.resolver_configs);
let mut writer = crate::dependency::DependencyWriter::new(&tx);
let mut resolved_here: std::collections::HashSet<i64> = std::collections::HashSet::new();
for (&k, &file_id) in row_k.iter().zip(&ids) {
let Some((imports, exports, declared)) =
read[k].as_ref().and_then(|f| f.imports.as_ref())
else {
continue;
};
let rel = &rels[k];
resolved_here.insert(file_id);
let deps = ctx.resolve_file_imports(file_id, rel, imports.clone(), &resolver);
writer.replace_dependencies(file_id, &deps)?;
writer.replace_members(
file_id,
&crate::dependency_resolve::package_members(rel, declared),
)?;
writer.clear_exports(file_id)?;
for export in exports {
let resolved = ctx.resolve_export(rel, export, &resolver);
writer.insert_export(
file_id,
export.exported_symbol.as_deref(),
&export.source_path,
resolved,
export.line_number,
)?;
}
}
let (deps_written, exports_written) = writer.counts();
let reresolved = if paths_changed {
reresolve(&tx, &ctx, &resolver, &resolved_here)?
} else {
0
};
refresh_vendored(&tx, &c.resolver_configs.vendor, Some(&ids))?;
laps.lap("deps");
crate::meta_update::set_statistic(
&tx,
RESOLVER_DIGEST_KEY,
&c.resolver_configs.digest,
now,
)?;
crate::meta_update::set_statistic(
&tx,
INDEX_GENERATION_KEY,
&c.generation.to_string(),
now,
)?;
CacheManager::update_branch_metadata_on(
&tx,
c.branch,
c.commit,
live_files,
flags.git_dirty,
)?;
CacheManager::set_total_files_on(&tx, live_files, now)?;
CacheManager::update_schema_hash_on(&tx)?;
CacheManager::update_extraction_hash_on(&tx)?;
laps.lap("stamps");
tx.commit()?;
self.abort_point("meta");
if let Ok(mut slot) = RESOLVER_CACHE.lock() {
*slot = Some((meta_path, manifest.publish_id, resolver));
}
laps.lap("commit");
self.cache
.checkpoint_wal()
.context("Failed to checkpoint WAL")?;
log::info!(
"dependencies: {} rows, {} exports written; {} rows re-resolved",
deps_written,
exports_written,
reresolved
);
log::info!(
"phase meta.db (files, branches, dependencies, exports): {} ms",
meta_start.elapsed().as_millis()
);
crate::query::invalidate_caches(c.root);
drop(snapshot);
crate::snapshot::remove_unreferenced(&cache_dir, &manifest, Some(&prev));
laps.lap("cleanup");
let (mut new_files, mut modified_files, mut unchanged_files) = c.breakdown_rest;
for (k, r) in read.iter().enumerate() {
if let Some(f) = r {
match c.branch_hashes.get(&rels[k]) {
None => new_files += 1,
Some(old) if *old != f.hash => modified_files += 1,
_ => unchanged_files += 1,
}
}
}
let mut stats = if c.full_stats {
self.cache.stats_synced()?
} else {
self.light_stats(live_files, now)
};
laps.lap("stats");
log::info!("delta update phases:{}", laps.line);
stats.new_files = new_files;
stats.modified_files = modified_files;
stats.deleted_files = c.deleted_file_count;
stats.unchanged_files = unchanged_files;
stats.skipped_too_large = c.skipped.0;
stats.skipped_bytes_too_large = c.skipped.1;
stats.skipped_binary = c.skipped.2;
Ok(Some(stats))
}
fn reuse_file(
&self,
ctx: &ProcessCtx<'_>,
snapshot: &crate::snapshot::IndexSnapshot,
id: u32,
i: usize,
meta: &Option<crate::cache::FileStat>,
trigram_scratch: &mut Vec<u64>,
) -> Option<FileProcessingResult> {
let row = ctx.stored.get(&ctx.rels[i])?;
let content = snapshot.get_file_content(id).ok()?.to_string();
let file_path = &ctx.root.join(&ctx.rels[i]);
let (size, mtime_ns) = match meta {
Some(md) if !row.stat_matches(md) => (md.size(), md.recorded_mtime_ns(ctx.run_start)),
_ => (row.size, row.mtime_ns),
};
let language = Language::from_path(file_path);
let line_count = content.lines().count();
let trigram_run = crate::trigram_build::extract_trigram_run(&content, trigram_scratch);
let imports = ctx.full_deps.then(|| {
let path_str = file_path.to_string_lossy().to_string();
extract_imports(language, &content, &path_str, ctx.root, ctx.tsconfigs)
});
Some(FileProcessingResult {
hash: row.hash.clone(),
content,
language,
line_count,
size,
mtime_ns,
imports,
trigram_run,
})
}
fn process_file(
&self,
ctx: &ProcessCtx<'_>,
i: usize,
trigram_scratch: &mut Vec<u64>,
) -> Option<FileProcessingResult> {
let file_path = &ctx.root.join(&ctx.rels[i]);
let path_str = file_path.to_string_lossy().to_string();
let (size, mtime_ns) = std::fs::metadata(file_path)
.map(|md| {
(
md.len(),
crate::cache::recorded_mtime_ns(&md, ctx.run_start),
)
})
.unwrap_or((0, 0));
let bytes = match std::fs::read(file_path) {
Ok(b) => b,
Err(e) => {
log::warn!("Failed to read {}: {}", path_str, e);
return None;
}
};
let hash = self.hash_content(&bytes);
let content = match String::from_utf8(bytes) {
Ok(s) => s,
Err(e) => String::from_utf8_lossy(e.as_bytes()).into_owned(),
};
let language = Language::from_path(file_path);
let line_count = content.lines().count();
let trigram_run = crate::trigram_build::extract_trigram_run(&content, trigram_scratch);
let unchanged = ctx
.stored
.get(&ctx.rels[i])
.is_some_and(|row| row.hash == hash);
let imports = (ctx.full_deps || !unchanged)
.then(|| extract_imports(language, &content, &path_str, ctx.root, ctx.tsconfigs));
Some(FileProcessingResult {
hash,
content,
language,
line_count,
size,
mtime_ns,
imports,
trigram_run,
})
}
fn run_marker(&self, cache_dir: &Path) -> std::time::SystemTime {
let marker = cache_dir.join(RUN_MARKER);
std::fs::write(&marker, b"")
.and_then(|_| std::fs::metadata(&marker))
.and_then(|md| md.modified())
.unwrap_or_else(|_| std::time::SystemTime::now())
}
fn stores_intact(&self, expected: usize, meta_generation: Option<u64>) -> bool {
match crate::snapshot::check_published(self.cache.path()) {
Ok(None) => {
log::info!("Index written before the manifest - rebuilding");
false
}
Ok(Some(manifest)) => {
if Some(manifest.generation) != meta_generation {
log::warn!(
"Manifest generation {} but meta.db rows are generation {:?} (a run stopped between the two) - rebuilding",
manifest.generation,
meta_generation
);
false
} else if manifest.live_files() as usize != expected {
log::warn!(
"Stores hold {} files but meta.db lists {} - rebuilding",
manifest.live_files(),
expected
);
false
} else {
true
}
}
Err(e) => {
log::warn!("Index stores unusable ({:#}) - rebuilding", e);
false
}
}
}
fn light_stats(&self, total_files: usize, now: i64) -> IndexStats {
let (index_size_bytes, trigram_index_bytes, corpus_bytes) = self.cache.store_sizes();
IndexStats {
total_files,
index_size_bytes,
last_updated: chrono::DateTime::from_timestamp(now, 0)
.unwrap_or_else(chrono::Utc::now)
.to_rfc3339(),
corpus_bytes,
trigram_index_bytes,
..Default::default()
}
}
fn refresh_named(&self, r: RefreshNamed<'_>) -> Result<IndexStats> {
let mut conn = crate::cache::open_meta_db(self.cache.path().join(crate::cache::META_DB))?;
let tx = conn
.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)
.context("Failed to begin meta.db transaction")?;
crate::meta_update::set_walk_seqs(&tx, r.walk_moves)?;
crate::meta_update::refresh_stats(&tx, r.touched)?;
crate::meta_update::set_dirty_flags(&tx, r.flips)?;
if r.commit.is_some() {
CacheManager::update_branch_metadata_on(
&tx,
r.branch,
r.commit,
r.live_files,
r.git_dirty,
)?;
}
CacheManager::update_stats_on(&tx, r.branch)?;
tx.commit()?;
let mut stats = self.light_stats(r.live_files, chrono::Utc::now().timestamp());
stats.new_files = 0;
stats.modified_files = 0;
stats.deleted_files = 0;
stats.unchanged_files = r.live_files;
stats.skipped_too_large = r.skipped.0;
stats.skipped_bytes_too_large = r.skipped.1;
stats.skipped_binary = r.skipped.2;
Ok(stats)
}
fn refresh_unchanged(&self, r: RefreshUnchanged<'_>) -> Result<IndexStats> {
let mut laps = Laps::new();
let now = chrono::Utc::now().timestamp();
let rows: Vec<&crate::meta_update::StoredFile> = r
.rows
.iter()
.map(|row| row.expect("nothing was added: every file has a row"))
.collect();
let seqs = crate::meta_update::plan_walk_seq(
&rows
.iter()
.map(|row| Some(row.walk_seq))
.collect::<Vec<_>>(),
);
let mut walk = Vec::new();
let mut flips = Vec::new();
let mut touched = Vec::new();
let no_dirty = r.dirty_paths.is_empty();
for (i, rel) in r.rels.iter().enumerate() {
let row = rows[i];
let dirty = !no_dirty && r.dirty_paths.contains(rel);
if seqs[i] != row.walk_seq {
walk.push((row.id, seqs[i]));
}
if r.status[i] == FileStatus::Touched {
let (size, mtime) = r.metas[i]
.as_ref()
.map(|md| (md.size(), md.recorded_mtime_ns(r.run_start)))
.unwrap_or((0, 0));
touched.push((row.id, size, mtime, dirty));
} else if dirty != row.dirty {
flips.push((row.id, dirty));
}
}
laps.lap("plan");
let mut conn = crate::cache::open_meta_db(self.cache.path().join(crate::cache::META_DB))?;
let tx = conn
.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)
.context("Failed to begin meta.db transaction")?;
crate::meta_update::set_walk_seqs(&tx, &walk)?;
crate::meta_update::refresh_stats(&tx, &touched)?;
crate::meta_update::set_dirty_flags(&tx, &flips)?;
let branch_id = self
.cache
.get_or_create_branch_id(&tx, r.branch, r.commit)?;
laps.lap("rows");
if !r.in_sync {
crate::meta_update::sync_branch_rows(&tx, branch_id, r.branch, now)?;
}
laps.lap("branch_rows");
if let Some(git_dirty) = r.git_dirty {
CacheManager::update_branch_metadata_on(
&tx,
r.branch,
r.commit,
r.rels.len(),
git_dirty,
)?;
}
CacheManager::set_total_files_on(&tx, r.rels.len(), now)?;
tx.commit()?;
laps.lap("commit");
let (new_files, modified_files, unchanged_files) = if r.in_sync {
(0, 0, rows.len())
} else {
breakdown(
r.rels
.iter()
.zip(&rows)
.map(|(rel, row)| (rel.as_str(), row.hash.as_str())),
r.existing_hashes,
)
};
let mut files_by_language: HashMap<String, usize> = HashMap::new();
let mut lines_by_language: HashMap<String, usize> = HashMap::new();
for row in &rows {
match files_by_language.get_mut(&row.language) {
Some(n) => *n += 1,
None => {
files_by_language.insert(row.language.clone(), 1);
}
}
match lines_by_language.get_mut(&row.language) {
Some(n) => *n += row.line_count,
None => {
lines_by_language.insert(row.language.clone(), row.line_count);
}
}
}
let (index_size_bytes, trigram_index_bytes, corpus_bytes) = self.cache.store_sizes();
let mut stats = IndexStats {
total_files: r.rels.len(),
index_size_bytes,
last_updated: chrono::DateTime::from_timestamp(now, 0)
.unwrap_or_else(chrono::Utc::now)
.to_rfc3339(),
files_by_language,
lines_by_language,
corpus_bytes,
trigram_index_bytes,
..Default::default()
};
laps.lap("stats");
log::info!("refresh phases:{}", laps.line);
stats.new_files = new_files;
stats.modified_files = modified_files;
stats.unchanged_files = unchanged_files;
stats.deleted_files = 0;
stats.skipped_too_large = r.skipped.0;
stats.skipped_bytes_too_large = r.skipped.1;
stats.skipped_binary = r.skipped.2;
Ok(stats)
}
fn write_meta(&self, w: MetaWrite<'_>) -> Result<()> {
let now = chrono::Utc::now().timestamp();
let mut laps = Laps::new();
let seqs = crate::meta_update::plan_walk_seq(
&w.present
.iter()
.map(|&i| w.stored.get(&w.rels[i]).map(|s| s.walk_seq))
.collect::<Vec<_>>(),
);
let written: HashMap<usize, &IndexedFile> =
w.written.iter().map(|f| (f.index, f)).collect();
let mut rows: Vec<crate::cache::FileRow> = Vec::new();
let mut row_files: Vec<usize> = Vec::new(); let mut walk = Vec::new();
let mut flips = Vec::new();
let mut touched = Vec::new();
for (k, &i) in w.present.iter().enumerate() {
let rel = &w.rels[i];
let dirty = w.dirty_paths.contains(rel);
let stored = w.stored.get(rel);
match written.get(&i) {
Some(f)
if !(w.status[i] == FileStatus::Unchanged
&& stored.is_some_and(|row| row.hash == f.hash)) =>
{
rows.push(crate::cache::FileRow {
path: rel.clone(),
hash: f.hash.clone(),
language: format!("{:?}", f.language),
line_count: f.line_count,
size: f.size,
mtime_ns: f.mtime_ns,
dirty,
walk_seq: seqs[k],
});
row_files.push(i);
}
_ => {
let Some(row) = stored else { continue };
if seqs[k] != row.walk_seq {
walk.push((row.id, seqs[k]));
}
if w.status[i] == FileStatus::Touched && !written.contains_key(&i) {
let (size, mtime) = w.metas[i]
.as_ref()
.map(|md| (md.size(), md.recorded_mtime_ns(w.run_start)))
.unwrap_or((0, 0));
touched.push((row.id, size, mtime, dirty));
} else if dirty != row.dirty {
flips.push((row.id, dirty));
}
}
}
}
let present: std::collections::HashSet<&str> =
w.present.iter().map(|&i| w.rels[i].as_str()).collect();
let deleted: Vec<i64> = w
.stored
.iter()
.filter(|(p, _)| !present.contains(p.as_str()))
.map(|(_, row)| row.id)
.collect();
let paths_changed = !deleted.is_empty()
|| w.present
.iter()
.any(|&i| !w.stored.contains_key(&w.rels[i]));
laps.lap("plan");
let mut conn = crate::cache::open_meta_db(self.cache.path().join(crate::cache::META_DB))?;
let tx = conn
.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)
.context("Failed to begin meta.db transaction")?;
crate::meta_update::delete_files(&tx, &deleted)?;
let new_ids = crate::meta_update::upsert_files(&tx, &rows, now)?;
crate::meta_update::set_walk_seqs(&tx, &walk)?;
crate::meta_update::refresh_stats(&tx, &touched)?;
crate::meta_update::set_dirty_flags(&tx, &flips)?;
laps.lap("rows");
let branch_id = self
.cache
.get_or_create_branch_id(&tx, w.branch, w.commit)?;
crate::meta_update::sync_branch_rows(&tx, branch_id, w.branch, now)?;
laps.lap("branch_rows");
log::info!(
"meta.db: {} rows written, {} deleted, {} moved, {} touched, {} dirty flags",
rows.len(),
deleted.len(),
walk.len(),
touched.len(),
flips.len()
);
let mut ids: HashMap<usize, i64> = w
.written
.iter()
.filter_map(|f| w.stored.get(&w.rels[f.index]).map(|s| (f.index, s.id)))
.collect();
for (i, id) in row_files.iter().zip(new_ids) {
ids.insert(*i, id);
}
let resolver = crate::dependency::PathResolver::from_conn(&tx)
.context("Failed to load file paths for dependency resolution")?;
laps.lap("resolver");
let ctx = crate::dependency_resolve::ResolverContext::new(w.root, w.resolver_configs);
let mut writer = crate::dependency::DependencyWriter::new(&tx);
if w.full_deps {
writer.clear_all()?;
}
let mut resolved_here: std::collections::HashSet<i64> = std::collections::HashSet::new();
for f in w.written {
let Some((imports, exports, declared)) = &f.imports else {
continue;
};
let rel = &w.rels[f.index];
let file_id = ids[&f.index];
resolved_here.insert(file_id);
let deps = ctx.resolve_file_imports(file_id, rel, imports.clone(), &resolver);
writer.replace_dependencies(file_id, &deps)?;
writer.replace_members(
file_id,
&crate::dependency_resolve::package_members(rel, declared),
)?;
if !w.full_deps {
writer.clear_exports(file_id)?;
}
for export in exports {
let resolved = ctx.resolve_export(rel, export, &resolver);
writer.insert_export(
file_id,
export.exported_symbol.as_deref(),
&export.source_path,
resolved,
export.line_number,
)?;
}
}
let (deps_written, exports_written) = writer.counts();
laps.lap("deps");
let mut reresolved = 0usize;
if !w.full_deps && paths_changed {
reresolved = reresolve(&tx, &ctx, &resolver, &resolved_here)?;
}
let written_ids: Vec<i64> = ids.values().copied().collect();
refresh_vendored(
&tx,
&w.resolver_configs.vendor,
(!w.full_deps).then_some(written_ids.as_slice()),
)?;
crate::meta_update::set_statistic(
&tx,
RESOLVER_DIGEST_KEY,
&w.resolver_configs.digest,
now,
)?;
crate::meta_update::set_statistic(
&tx,
INDEX_GENERATION_KEY,
&w.generation.to_string(),
now,
)?;
laps.lap("reresolve");
tx.commit()?;
laps.lap("commit");
log::info!("meta.db write phases:{}", laps.line);
log::info!(
"dependencies: {} rows, {} exports written; {} rows re-resolved",
deps_written,
exports_written,
reresolved
);
Ok(())
}
#[cfg(test)]
fn discover_files(
&self,
root: &Path,
stored: &HashMap<String, crate::meta_update::StoredFile>,
) -> Result<Discovered> {
self.discover_files_in(root, stored, None)
}
fn discover_files_in(
&self,
root: &Path,
stored: &HashMap<String, crate::meta_update::StoredFile>,
targets: Option<Arc<Targets>>,
) -> Result<Discovered> {
let walked = self.walk_candidates(root, targets)?;
Ok(Self::drop_binaries(root, walked, stored, None))
}
fn walk_candidates(&self, root: &Path, targets: Option<Arc<Targets>>) -> Result<Walked> {
let mut out = Walked::default();
let policy = self.path_policy(root);
let walker = Self::targeted_walk_builder(root, &self.config, &policy, targets).build();
for entry in walker {
let entry = entry?;
let path = entry.path();
if !entry.file_type().map(|ft| ft.is_file()).unwrap_or(false) {
continue;
}
let Some(lang) = policy.classify(path) else {
continue;
};
let mut size = 0u64;
let metadata = std::fs::metadata(path).ok();
if let Some(metadata) = &metadata {
size = metadata.len();
if size > self.config.max_file_size as u64 {
log::debug!("Skipping {} (too large: {} bytes)", path.display(), size);
out.found.skipped_too_large += 1;
out.found.skipped_bytes_too_large += size;
continue;
}
}
out.found.rels.push(normalize_rel(root, path));
out.found.sizes.push(size);
out.found
.metas
.push(metadata.as_ref().map(crate::cache::FileStat::of));
out.non_code.push(!lang.is_code());
}
Ok(out)
}
fn drop_binaries(
root: &Path,
walked: Walked,
stored: &HashMap<String, crate::meta_update::StoredFile>,
pool: Option<&rayon::ThreadPool>,
) -> Discovered {
let Walked {
found: mut out,
non_code,
} = walked;
let to_sniff: Vec<usize> = (0..out.rels.len())
.filter(|&i| {
non_code[i]
&& !stored
.get(&out.rels[i])
.zip(out.metas[i].as_ref())
.is_some_and(|(row, md)| row.stat_matches(md))
})
.collect();
let sniff = |i: &usize| looks_binary(&root.join(&out.rels[*i]));
let binary: Vec<bool> = match pool {
Some(pool) => pool.install(|| to_sniff.par_iter().map(sniff).collect()),
None => to_sniff.iter().map(sniff).collect(),
};
let dropped: std::collections::HashSet<usize> = to_sniff
.iter()
.zip(binary)
.filter_map(|(&i, b)| b.then_some(i))
.collect();
if dropped.is_empty() {
return out;
}
for &i in &dropped {
log::debug!("Skipping {} (binary)", out.rels[i]);
}
out.skipped_binary += dropped.len();
fn keep<T>(v: &mut Vec<T>, dropped: &std::collections::HashSet<usize>) {
let mut i = 0;
v.retain(|_| {
let k = !dropped.contains(&i);
i += 1;
k
});
}
keep(&mut out.rels, &dropped);
keep(&mut out.sizes, &dropped);
keep(&mut out.metas, &dropped);
out
}
fn targeted_walk_builder(
root: &Path,
config: &IndexConfig,
policy: &PathPolicy,
targets: Option<Arc<Targets>>,
) -> WalkBuilder {
let mut builder = Self::walk_builder(root, config, policy);
if let Some(targets) = targets {
let hidden = policy.hidden();
let root_buf = root.to_path_buf();
builder.filter_entry(move |e| {
if hidden {
let name = e.file_name();
if name == ".git" || name == crate::cache::CACHE_DIR {
return false;
}
}
let rel = normalize_rel(&root_buf, e.path());
targets.admits(&rel)
});
}
builder
}
pub(crate) fn walked_among(
root: &Path,
config: &IndexConfig,
policy: &PathPolicy,
rels: &[String],
) -> std::collections::HashSet<String> {
let targets = Arc::new(Targets::new(rels));
let walker =
Self::targeted_walk_builder(root, config, policy, Some(Arc::clone(&targets))).build();
walker
.flatten()
.filter(|e| e.file_type().is_some_and(|t| t.is_file()))
.map(|e| normalize_rel(root, e.path()))
.filter(|rel| targets.exact.contains(rel))
.collect()
}
pub fn walk_builder(root: &Path, config: &IndexConfig, policy: &PathPolicy) -> WalkBuilder {
let mut builder = WalkBuilder::new(root);
builder
.follow_links(config.follow_symlinks)
.hidden(!policy.hidden())
.git_ignore(true) .git_global(false) .git_exclude(false); if policy.hidden() {
builder.filter_entry(|e| {
let name = e.file_name();
name != ".git" && name != crate::cache::CACHE_DIR
});
}
if let Some(ov) = policy.overrides() {
builder.overrides(ov.clone());
}
builder
}
pub fn is_indexable_path(path: &Path) -> bool {
Self::is_indexable_path_with(path, None)
}
pub fn is_indexable_path_with(path: &Path, policy: Option<&PathPolicy>) -> bool {
let default_policy;
let policy = match policy {
Some(p) => p,
None => {
default_policy = PathPolicy::default();
&default_policy
}
};
policy.hidden_ok(path) && policy.classify(path).is_some()
}
fn should_index_lang(&self, path: &Path) -> bool {
PathPolicy::from_config(Path::new("."), &self.config)
.classify(path)
.is_some()
}
#[allow(dead_code)]
fn should_index(&self, path: &Path) -> bool {
if !self.should_index_lang(path) {
return false;
}
if let Ok(metadata) = std::fs::metadata(path)
&& metadata.len() > self.config.max_file_size as u64
{
log::debug!(
"Skipping {} (too large: {} bytes)",
path.display(),
metadata.len()
);
return false;
}
true
}
fn hash_content(&self, content: &[u8]) -> String {
let hash = blake3::hash(content);
hash.to_hex().to_string()
}
#[cfg_attr(not(unix), allow(unused_variables))]
fn check_disk_space(&self, root: &Path) -> Result<()> {
let cache_path = self.cache.path();
#[cfg(unix)]
{
let test_file = cache_path.join(".space_check");
match std::fs::write(&test_file, b"test") {
Ok(_) => {
let _ = std::fs::remove_file(&test_file);
if let Ok(output) = std::process::Command::new("df")
.arg("-k")
.arg(cache_path.parent().unwrap_or(root))
.output()
&& let Ok(df_output) = String::from_utf8(output.stdout)
{
if let Some(line) = df_output.lines().nth(1) {
let parts: Vec<&str> = line.split_whitespace().collect();
if parts.len() >= 4
&& let Ok(available_kb) = parts[3].parse::<u64>()
{
let available_mb = available_kb / 1024;
if available_mb < 100 {
log::warn!(
"Low disk space: only {}MB available. Indexing may fail.",
available_mb
);
output::warn(&format!(
"Low disk space ({}MB available). Consider freeing up space.",
available_mb
));
} else {
log::debug!("Available disk space: {}MB", available_mb);
}
}
}
}
Ok(())
}
Err(e) if e.kind() == std::io::ErrorKind::PermissionDenied => {
anyhow::bail!(
"Permission denied writing to cache directory: {}. Check file permissions.",
cache_path.display()
)
}
Err(e) => {
log::warn!(
"Failed to write test file (possible disk space issue): {}",
e
);
Err(e).context(
"Failed to verify disk space - indexing may fail due to insufficient space",
)
}
}
}
#[cfg(not(unix))]
{
let test_file = cache_path.join(".space_check");
match std::fs::write(&test_file, b"test") {
Ok(_) => {
let _ = std::fs::remove_file(&test_file);
Ok(())
}
Err(e) if e.kind() == std::io::ErrorKind::PermissionDenied => {
anyhow::bail!(
"Permission denied writing to cache directory: {}. Check file permissions.",
cache_path.display()
)
}
Err(e) => {
log::warn!(
"Failed to write test file (possible disk space issue): {}",
e
);
Err(e).context(
"Failed to verify disk space - indexing may fail due to insufficient space",
)
}
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::fs;
use tempfile::TempDir;
#[test]
fn test_indexer_creation() {
let temp = TempDir::new().unwrap();
let cache = CacheManager::new(temp.path());
let config = IndexConfig::default();
let indexer = Indexer::new(cache, config);
assert!(indexer.cache.path().ends_with(".reflex"));
}
#[test]
fn test_hash_content() {
let temp = TempDir::new().unwrap();
let cache = CacheManager::new(temp.path());
let config = IndexConfig::default();
let indexer = Indexer::new(cache, config);
let content1 = b"hello world";
let content2 = b"hello world";
let content3 = b"different content";
let hash1 = indexer.hash_content(content1);
let hash2 = indexer.hash_content(content2);
let hash3 = indexer.hash_content(content3);
assert_eq!(hash1, hash2);
assert_ne!(hash1, hash3);
assert_eq!(hash1.len(), 64); }
#[test]
fn test_should_index_rust_file() {
let temp = TempDir::new().unwrap();
let cache = CacheManager::new(temp.path());
let config = IndexConfig::default();
let indexer = Indexer::new(cache, config);
let rust_file = temp.path().join("test.rs");
fs::write(&rust_file, "fn main() {}").unwrap();
assert!(indexer.should_index(&rust_file));
}
#[test]
fn test_should_index_unsupported_extension() {
let temp = TempDir::new().unwrap();
let cache = CacheManager::new(temp.path());
let config = IndexConfig::default();
let indexer = Indexer::new(cache, config);
let unsupported_file = temp.path().join("test.xyz");
fs::write(&unsupported_file, "mystery format").unwrap();
assert!(indexer.should_index(&unsupported_file));
let allowlist = Indexer::new(
CacheManager::new(temp.path()),
IndexConfig {
mode: IndexMode::Allowlist,
..Default::default()
},
);
assert!(!allowlist.should_index(&unsupported_file));
let binary_file = temp.path().join("logo.png");
fs::write(&binary_file, "not really a png").unwrap();
assert!(!allowlist.should_index(&binary_file));
}
#[test]
fn test_binary_sniff() {
assert!(!is_binary(b"plain text\n"));
assert!(is_binary(b"\x89PNG\r\n\x1a\n\0\0"));
let mut late = vec![b'a'; 64 * 1024];
late.push(0);
assert!(is_binary(&late));
}
#[test]
fn test_should_index_text_tier() {
let temp = TempDir::new().unwrap();
let indexer = Indexer::new(CacheManager::new(temp.path()), IndexConfig::default());
for name in [
"notes.md",
"config.yaml",
"data.json",
"api.proto",
"run.sh",
] {
let path = temp.path().join(name);
fs::write(&path, "realm_marker").unwrap();
assert!(indexer.should_index(&path), "{name} should be indexed");
}
let lock = temp.path().join("package-lock.json");
fs::write(&lock, "{}").unwrap();
assert!(indexer.should_index(&lock));
let allowlist = Indexer::new(
CacheManager::new(temp.path()),
IndexConfig {
mode: IndexMode::Allowlist,
..Default::default()
},
);
assert!(!allowlist.should_index(&lock));
}
#[test]
fn test_text_tier_can_be_disabled() {
let temp = TempDir::new().unwrap();
let indexer = Indexer::new(
CacheManager::new(temp.path()),
IndexConfig {
text_tier: false,
..Default::default()
},
);
let md = temp.path().join("notes.md");
fs::write(&md, "text").unwrap();
assert!(!indexer.should_index(&md));
let rs = temp.path().join("main.rs");
fs::write(&rs, "fn main() {}").unwrap();
assert!(indexer.should_index(&rs), "code is unaffected");
}
#[test]
fn test_should_index_no_extension() {
let temp = TempDir::new().unwrap();
let cache = CacheManager::new(temp.path());
let config = IndexConfig::default();
let indexer = Indexer::new(cache, config);
let makefile = temp.path().join("Makefile");
fs::write(&makefile, "all:\n\techo hello").unwrap();
assert!(indexer.should_index(&makefile));
let readme = temp.path().join("README");
fs::write(&readme, "hello").unwrap();
assert!(indexer.should_index(&readme));
let allowlist = Indexer::new(
CacheManager::new(temp.path()),
IndexConfig {
mode: IndexMode::Allowlist,
..Default::default()
},
);
assert!(allowlist.should_index(&makefile));
assert!(!allowlist.should_index(&readme));
}
#[test]
fn test_should_index_size_limit() {
let temp = TempDir::new().unwrap();
let cache = CacheManager::new(temp.path());
let config = IndexConfig {
max_file_size: 100,
..Default::default()
};
let indexer = Indexer::new(cache, config);
let small_file = temp.path().join("small.rs");
fs::write(&small_file, "fn main() {}").unwrap();
assert!(indexer.should_index(&small_file));
let large_file = temp.path().join("large.rs");
let large_content = "a".repeat(150);
fs::write(&large_file, large_content).unwrap();
assert!(!indexer.should_index(&large_file));
}
#[test]
fn test_discover_files_empty_dir() {
let temp = TempDir::new().unwrap();
let cache = CacheManager::new(temp.path());
let config = IndexConfig::default();
let indexer = Indexer::new(cache, config);
let files = indexer
.discover_files(temp.path(), &HashMap::new())
.unwrap()
.rels
.iter()
.map(std::path::PathBuf::from)
.collect::<Vec<_>>();
assert_eq!(files.len(), 0);
}
#[test]
fn test_discover_files_single_file() {
let temp = TempDir::new().unwrap();
let cache = CacheManager::new(temp.path());
let config = IndexConfig::default();
let indexer = Indexer::new(cache, config);
let rust_file = temp.path().join("main.rs");
fs::write(&rust_file, "fn main() {}").unwrap();
let files = indexer
.discover_files(temp.path(), &HashMap::new())
.unwrap()
.rels
.iter()
.map(std::path::PathBuf::from)
.collect::<Vec<_>>();
assert_eq!(files.len(), 1);
assert!(files[0].ends_with("main.rs"));
}
#[test]
fn test_discover_files_multiple_languages() {
let temp = TempDir::new().unwrap();
let cache = CacheManager::new(temp.path());
let config = IndexConfig::default();
let indexer = Indexer::new(cache, config);
fs::write(temp.path().join("main.rs"), "fn main() {}").unwrap();
fs::write(temp.path().join("script.py"), "print('hello')").unwrap();
fs::write(temp.path().join("app.js"), "console.log('hi')").unwrap();
fs::write(temp.path().join("README.md"), "# Project").unwrap();
fs::write(temp.path().join("mystery.xyz"), "?").unwrap();
fs::write(temp.path().join("blob.bin"), b"\0\x01\x02").unwrap();
let found = indexer
.discover_files(temp.path(), &HashMap::new())
.unwrap();
assert_eq!(found.rels.len(), 5, "3 code files, the markdown, the .xyz");
assert_eq!(found.skipped_binary, 1);
}
#[test]
fn test_discover_files_subdirectories() {
let temp = TempDir::new().unwrap();
let cache = CacheManager::new(temp.path());
let config = IndexConfig::default();
let indexer = Indexer::new(cache, config);
let src_dir = temp.path().join("src");
fs::create_dir(&src_dir).unwrap();
fs::write(src_dir.join("main.rs"), "fn main() {}").unwrap();
fs::write(src_dir.join("lib.rs"), "pub mod test {}").unwrap();
let tests_dir = temp.path().join("tests");
fs::create_dir(&tests_dir).unwrap();
fs::write(tests_dir.join("test.rs"), "#[test] fn test() {}").unwrap();
let files = indexer
.discover_files(temp.path(), &HashMap::new())
.unwrap()
.rels
.iter()
.map(std::path::PathBuf::from)
.collect::<Vec<_>>();
assert_eq!(files.len(), 3);
}
#[test]
fn test_discover_files_respects_gitignore() {
let temp = TempDir::new().unwrap();
std::process::Command::new("git")
.arg("init")
.current_dir(temp.path())
.output()
.expect("Failed to initialize git repo");
let cache = CacheManager::new(temp.path());
let config = IndexConfig::default();
let indexer = Indexer::new(cache, config);
fs::write(temp.path().join(".gitignore"), "ignored/\n").unwrap();
fs::write(temp.path().join("included.rs"), "fn main() {}").unwrap();
fs::write(temp.path().join("also_included.py"), "print('hi')").unwrap();
let ignored_dir = temp.path().join("ignored");
fs::create_dir(&ignored_dir).unwrap();
fs::write(ignored_dir.join("excluded.rs"), "fn test() {}").unwrap();
let files = indexer
.discover_files(temp.path(), &HashMap::new())
.unwrap()
.rels
.iter()
.map(std::path::PathBuf::from)
.collect::<Vec<_>>();
assert!(
files.iter().any(|f| f.ends_with("included.rs")),
"Should find included.rs"
);
assert!(
files.iter().any(|f| f.ends_with("also_included.py")),
"Should find also_included.py"
);
assert!(
!files.iter().any(|f| {
let path_str = f.to_string_lossy();
path_str.contains("ignored") && f.ends_with("excluded.rs")
}),
"Should NOT find excluded.rs in ignored/ directory (gitignore pattern)"
);
assert_eq!(
files.len(),
2,
"Should find exactly 2 files (not including .gitignore or ignored/excluded.rs)"
);
}
#[test]
fn test_index_empty_directory() {
let temp = TempDir::new().unwrap();
let cache = CacheManager::new(temp.path());
let config = IndexConfig::default();
let indexer = Indexer::new(cache, config);
let stats = indexer.index(temp.path(), false).unwrap();
assert_eq!(stats.total_files, 0);
}
#[test]
fn test_index_single_rust_file() {
let temp = TempDir::new().unwrap();
let project_root = temp.path().join("project");
fs::create_dir(&project_root).unwrap();
let cache = CacheManager::new(&project_root);
let config = IndexConfig::default();
let indexer = Indexer::new(cache, config);
fs::write(
project_root.join("main.rs"),
"fn main() { println!(\"Hello\"); }",
)
.unwrap();
let stats = indexer.index(&project_root, false).unwrap();
assert_eq!(stats.total_files, 1);
assert!(stats.files_by_language.contains_key("Rust"));
}
#[test]
fn test_index_multiple_files() {
let temp = TempDir::new().unwrap();
let project_root = temp.path().join("project");
fs::create_dir(&project_root).unwrap();
let cache = CacheManager::new(&project_root);
let config = IndexConfig::default();
let indexer = Indexer::new(cache, config);
fs::write(project_root.join("main.rs"), "fn main() {}").unwrap();
fs::write(project_root.join("lib.rs"), "pub fn test() {}").unwrap();
fs::write(project_root.join("script.py"), "def main(): pass").unwrap();
let stats = indexer.index(&project_root, false).unwrap();
assert_eq!(stats.total_files, 3);
assert_eq!(stats.files_by_language.get("Rust"), Some(&2));
assert_eq!(stats.files_by_language.get("Python"), Some(&1));
}
#[test]
fn test_index_creates_trigram_index() {
let temp = TempDir::new().unwrap();
let project_root = temp.path().join("project");
fs::create_dir(&project_root).unwrap();
let cache = CacheManager::new(&project_root);
let config = IndexConfig::default();
let indexer = Indexer::new(cache, config);
fs::write(project_root.join("main.rs"), "fn main() {}").unwrap();
indexer.index(&project_root, false).unwrap();
let trigrams_path = project_root.join(".reflex/trigrams.bin");
assert!(trigrams_path.exists());
}
#[test]
fn test_index_creates_content_store() {
let temp = TempDir::new().unwrap();
let project_root = temp.path().join("project");
fs::create_dir(&project_root).unwrap();
let cache = CacheManager::new(&project_root);
let config = IndexConfig::default();
let indexer = Indexer::new(cache, config);
fs::write(project_root.join("main.rs"), "fn main() {}").unwrap();
indexer.index(&project_root, false).unwrap();
let content_path = project_root.join(".reflex/content.bin");
assert!(content_path.exists());
}
#[test]
fn test_index_incremental_no_changes() {
let temp = TempDir::new().unwrap();
let project_root = temp.path().join("project");
fs::create_dir(&project_root).unwrap();
let cache = CacheManager::new(&project_root);
let config = IndexConfig::default();
let indexer = Indexer::new(cache, config);
fs::write(project_root.join("main.rs"), "fn main() {}").unwrap();
let stats1 = indexer.index(&project_root, false).unwrap();
assert_eq!(stats1.total_files, 1);
let stats2 = indexer.index(&project_root, false).unwrap();
assert_eq!(stats2.total_files, 1);
}
#[test]
fn test_index_incremental_with_changes() {
let temp = TempDir::new().unwrap();
let project_root = temp.path().join("project");
fs::create_dir(&project_root).unwrap();
let cache = CacheManager::new(&project_root);
let config = IndexConfig::default();
let indexer = Indexer::new(cache, config);
let main_path = project_root.join("main.rs");
fs::write(&main_path, "fn main() {}").unwrap();
indexer.index(&project_root, false).unwrap();
fs::write(&main_path, "fn main() { println!(\"changed\"); }").unwrap();
let stats = indexer.index(&project_root, false).unwrap();
assert_eq!(stats.total_files, 1);
}
#[test]
fn test_index_incremental_new_file() {
let temp = TempDir::new().unwrap();
let project_root = temp.path().join("project");
fs::create_dir(&project_root).unwrap();
let cache = CacheManager::new(&project_root);
let config = IndexConfig::default();
let indexer = Indexer::new(cache, config);
fs::write(project_root.join("main.rs"), "fn main() {}").unwrap();
let stats1 = indexer.index(&project_root, false).unwrap();
assert_eq!(stats1.total_files, 1);
fs::write(project_root.join("lib.rs"), "pub fn test() {}").unwrap();
let stats2 = indexer.index(&project_root, false).unwrap();
assert_eq!(stats2.total_files, 2);
}
#[test]
fn test_index_parallel_threads_config() {
let temp = TempDir::new().unwrap();
let project_root = temp.path().join("project");
fs::create_dir(&project_root).unwrap();
let cache = CacheManager::new(&project_root);
let config = IndexConfig {
parallel_threads: 2,
..Default::default()
};
let indexer = Indexer::new(cache, config);
fs::write(project_root.join("main.rs"), "fn main() {}").unwrap();
let stats = indexer.index(&project_root, false).unwrap();
assert_eq!(stats.total_files, 1);
}
#[test]
fn test_index_parallel_threads_auto() {
let temp = TempDir::new().unwrap();
let project_root = temp.path().join("project");
fs::create_dir(&project_root).unwrap();
let cache = CacheManager::new(&project_root);
let config = IndexConfig {
parallel_threads: 0,
..Default::default()
};
let indexer = Indexer::new(cache, config);
fs::write(project_root.join("main.rs"), "fn main() {}").unwrap();
let stats = indexer.index(&project_root, false).unwrap();
assert_eq!(stats.total_files, 1);
}
#[test]
fn test_index_respects_size_limit() {
let temp = TempDir::new().unwrap();
let project_root = temp.path().join("project");
fs::create_dir(&project_root).unwrap();
let cache = CacheManager::new(&project_root);
let config = IndexConfig {
max_file_size: 50,
..Default::default()
};
let indexer = Indexer::new(cache, config);
fs::write(project_root.join("small.rs"), "fn a() {}").unwrap();
let large_content = "fn main() {}\n".repeat(10);
fs::write(project_root.join("large.rs"), large_content).unwrap();
let stats = indexer.index(&project_root, false).unwrap();
assert_eq!(stats.total_files, 1);
}
#[test]
fn test_index_mixed_languages() {
let temp = TempDir::new().unwrap();
let project_root = temp.path().join("project");
fs::create_dir(&project_root).unwrap();
let cache = CacheManager::new(&project_root);
let config = IndexConfig::default();
let indexer = Indexer::new(cache, config);
fs::write(project_root.join("main.rs"), "fn main() {}").unwrap();
fs::write(project_root.join("test.py"), "def test(): pass").unwrap();
fs::write(project_root.join("app.js"), "function main() {}").unwrap();
fs::write(project_root.join("lib.go"), "func main() {}").unwrap();
let stats = indexer.index(&project_root, false).unwrap();
assert_eq!(stats.total_files, 4);
assert!(stats.files_by_language.contains_key("Rust"));
assert!(stats.files_by_language.contains_key("Python"));
assert!(stats.files_by_language.contains_key("JavaScript"));
assert!(stats.files_by_language.contains_key("Go"));
}
#[test]
fn test_index_updates_cache_stats() {
let temp = TempDir::new().unwrap();
let project_root = temp.path().join("project");
fs::create_dir(&project_root).unwrap();
let cache = CacheManager::new(&project_root);
let config = IndexConfig::default();
let indexer = Indexer::new(cache, config);
fs::write(project_root.join("main.rs"), "fn main() {}").unwrap();
indexer.index(&project_root, false).unwrap();
let cache = CacheManager::new(&project_root);
let stats = cache.stats().unwrap();
assert_eq!(stats.total_files, 1);
assert!(stats.index_size_bytes > 0);
}
}