use anyhow::{Context, Result};
use rayon::prelude::*;
use serde::{Deserialize, Serialize};
use std::fs::File;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::time::Instant;
use crate::cache::CacheManager;
use crate::content_store::ContentReader;
use crate::parsers::ParserFactory;
use crate::symbol_cache::SymbolCache;
const LOCK_FILE: &str = "indexing.lock";
const LOCK_MAX_AGE: std::time::Duration = std::time::Duration::from_secs(900);
const STATUS_FILE: &str = "indexing.status";
const HEARTBEAT_MAX_AGE: std::time::Duration = std::time::Duration::from_secs(60);
const CANCEL_FILE: &str = "indexing.cancel";
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LockHolder {
pub pid: u32,
#[serde(default)]
pub started_at: Option<String>,
}
impl LockHolder {
pub fn started_clock(&self) -> String {
self.started_at
.as_deref()
.and_then(|s| chrono::DateTime::parse_from_rfc3339(s).ok())
.map(|dt| {
dt.with_timezone(&chrono::Local)
.format("%H:%M:%S")
.to_string()
})
.unwrap_or_else(|| "unknown".to_string())
}
}
fn pid_is_live_symbol_pass(pid: u32) -> Option<bool> {
if pid == std::process::id() {
return Some(true);
}
#[cfg(target_os = "linux")]
{
let proc_dir = std::path::PathBuf::from(format!("/proc/{}", pid));
if !proc_dir.exists() {
return Some(false);
}
match std::fs::read(proc_dir.join("cmdline")) {
Ok(raw) => Some(String::from_utf8_lossy(&raw).contains("index-symbols-internal")),
Err(_) => Some(true),
}
}
#[cfg(target_os = "macos")]
{
let out = std::process::Command::new("ps")
.args(["-o", "command=", "-p", &pid.to_string()])
.output()
.ok()?;
if !out.status.success() {
return Some(false);
}
Some(String::from_utf8_lossy(&out.stdout).contains("index-symbols-internal"))
}
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
{
let _ = pid;
None
}
}
pub const fn pid_liveness_supported() -> bool {
cfg!(any(target_os = "linux", target_os = "macos"))
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct IndexingStatus {
pub state: IndexerState,
pub total_files: usize,
pub processed_files: usize,
pub cached_files: usize,
pub parsed_files: usize,
pub failed_files: usize,
pub started_at: String,
pub updated_at: String,
pub completed_at: Option<String>,
pub error: Option<String>,
#[serde(default)]
pub pid: u32,
#[serde(default)]
pub phase: String,
#[serde(default)]
pub current_file: Option<String>,
#[serde(default)]
pub write_failed_files: usize,
#[serde(default)]
pub skipped_minified: usize,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "lowercase")]
pub enum IndexerState {
Running,
Completed,
Failed,
Cancelled,
}
fn is_lock_stale(lock_path: &Path) -> bool {
is_lock_stale_by(lock_path, LOCK_MAX_AGE)
}
fn is_lock_stale_by(lock_path: &Path, max_age: std::time::Duration) -> bool {
let metadata = match std::fs::metadata(lock_path) {
Ok(m) => m,
Err(_) => return false, };
let modified = match metadata.modified() {
Ok(t) => t,
Err(_) => return false,
};
match modified.elapsed() {
Ok(age) => age > max_age,
Err(_) => false, }
}
enum ParseOutcome {
Parsed(Vec<crate::models::SearchResult>),
SkippedMinified,
}
pub struct BackgroundIndexer {
workspace_path: PathBuf,
cache_path: PathBuf,
status: IndexingStatus,
}
impl BackgroundIndexer {
pub fn new(workspace_path: &Path) -> Result<Self> {
let now = chrono::Utc::now().to_rfc3339();
let cache_mgr = CacheManager::new(workspace_path);
let cache_path = cache_mgr.path().to_path_buf();
Ok(Self {
workspace_path: workspace_path.to_path_buf(),
cache_path,
status: IndexingStatus {
state: IndexerState::Running,
total_files: 0,
processed_files: 0,
cached_files: 0,
parsed_files: 0,
failed_files: 0,
started_at: now.clone(),
updated_at: now,
completed_at: None,
error: None,
pid: std::process::id(),
phase: "starting".to_string(),
current_file: None,
write_failed_files: 0,
skipped_minified: 0,
},
})
}
pub fn lock_holder(cache_dir: &Path) -> Option<LockHolder> {
let lock_path = cache_dir.join(LOCK_FILE);
let raw = std::fs::read_to_string(&lock_path).ok()?;
let holder: LockHolder = serde_json::from_str(raw.trim()).unwrap_or_else(|_| LockHolder {
pid: raw.trim().parse().unwrap_or(0),
started_at: None,
});
let stale = match pid_is_live_symbol_pass(holder.pid) {
Some(true) => false,
Some(false) => {
log::warn!(
"Reaping indexing lock: pid {} is not a live symbol pass",
holder.pid
);
true
}
None => {
if is_lock_stale(&lock_path) {
log::warn!("Removing indexing lock older than {:?}", LOCK_MAX_AGE);
true
} else if !Self::heartbeat_is_fresh(cache_dir)
&& is_lock_stale_by(&lock_path, HEARTBEAT_MAX_AGE)
{
log::warn!(
"Removing indexing lock: no heartbeat within {:?} and liveness \
is not determinable on this platform",
HEARTBEAT_MAX_AGE
);
true
} else {
false
}
}
};
if stale {
let _ = std::fs::remove_file(&lock_path);
return None;
}
Some(holder)
}
pub fn heartbeat_is_fresh(cache_dir: &Path) -> bool {
let Ok(Some(status)) = Self::get_status(cache_dir) else {
return false;
};
if status.state != IndexerState::Running {
return false;
}
let Ok(updated) = chrono::DateTime::parse_from_rfc3339(&status.updated_at) else {
return false;
};
let age = chrono::Utc::now().signed_duration_since(updated.with_timezone(&chrono::Utc));
age < chrono::Duration::from_std(HEARTBEAT_MAX_AGE).unwrap_or(chrono::Duration::zero())
}
pub fn is_running(cache_dir: &Path) -> bool {
Self::lock_holder(cache_dir).is_some()
}
pub fn request_cancel(cache_dir: &Path) -> std::io::Result<()> {
std::fs::write(cache_dir.join(CANCEL_FILE), b"")
}
pub fn cancel_requested(cache_dir: &Path) -> bool {
cache_dir.join(CANCEL_FILE).exists()
}
pub fn clear_cancel(cache_dir: &Path) {
let _ = std::fs::remove_file(cache_dir.join(CANCEL_FILE));
}
pub fn get_status(cache_dir: &Path) -> Result<Option<IndexingStatus>> {
let status_path = cache_dir.join(STATUS_FILE);
if !status_path.exists() {
return Ok(None);
}
let status_json =
std::fs::read_to_string(&status_path).context("Failed to read indexing status")?;
let status: IndexingStatus =
serde_json::from_str(&status_json).context("Failed to parse indexing status")?;
Ok(Some(status))
}
fn acquire_lock(&self) -> Result<File> {
let lock_path = self.cache_path.join(LOCK_FILE);
if Self::lock_holder(&self.cache_path).is_some() {
anyhow::bail!("Indexing already in progress (lock file exists)");
}
let mut lock_file = File::create(&lock_path).context("Failed to create lock file")?;
let pid = std::process::id();
let holder = LockHolder {
pid,
started_at: Some(self.status.started_at.clone()),
};
writeln!(lock_file, "{}", serde_json::to_string(&holder)?)?;
lock_file.flush()?;
log::debug!("Acquired indexing lock (PID: {})", pid);
Ok(lock_file)
}
fn release_lock(&self) -> Result<()> {
let lock_path = self.cache_path.join(LOCK_FILE);
if lock_path.exists() {
std::fs::remove_file(&lock_path).context("Failed to remove lock file")?;
log::debug!("Released indexing lock");
}
Ok(())
}
fn write_status(&mut self) -> Result<()> {
self.status.updated_at = chrono::Utc::now().to_rfc3339();
let status_path = self.cache_path.join(STATUS_FILE);
let status_json =
serde_json::to_string_pretty(&self.status).context("Failed to serialize status")?;
std::fs::write(&status_path, status_json).context("Failed to write status file")?;
Ok(())
}
pub fn run(&mut self) -> Result<()> {
let start_time = Instant::now();
Self::clear_cancel(&self.cache_path);
let _lock_file = self
.acquire_lock()
.context("Failed to acquire indexing lock")?;
let cache_path = self.cache_path.clone();
let _guard = scopeguard::guard((), move |_| {
let _ = std::fs::remove_file(cache_path.join(LOCK_FILE));
});
let result = self.run_internal();
match result {
Ok(()) if self.status.state == IndexerState::Cancelled => {
self.status.completed_at = Some(chrono::Utc::now().to_rfc3339());
log::info!(
"Symbol indexing cancelled after {} of {} files in {:.2}s",
self.status.processed_files,
self.status.total_files,
start_time.elapsed().as_secs_f64()
);
}
Ok(()) => {
self.status.state = IndexerState::Completed;
self.status.phase = "done".to_string();
self.status.completed_at = Some(chrono::Utc::now().to_rfc3339());
log::info!(
"Symbol indexing completed: {} files processed ({} cached, {} parsed, {} failed) in {:.2}s",
self.status.processed_files,
self.status.cached_files,
self.status.parsed_files,
self.status.failed_files,
start_time.elapsed().as_secs_f64()
);
}
Err(ref e) => {
self.status.state = IndexerState::Failed;
self.status.error = Some(format!("{:#}", e));
self.status.completed_at = Some(chrono::Utc::now().to_rfc3339());
log::error!("Symbol indexing failed: {:#}", e);
}
}
self.write_status()?;
self.release_lock()?;
result
}
fn run_internal(&mut self) -> Result<()> {
log::info!("Starting background symbol indexing");
let cache_mgr = CacheManager::new(&self.workspace_path);
let config = cache_mgr.load_index_config().unwrap_or_default();
let num_threads = crate::models::resolve_symbol_thread_count(config.symbol_threads);
log::info!(
"Using {} threads for background indexing ({} CPUs available)",
num_threads,
std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(1)
);
let thread_pool = rayon::ThreadPoolBuilder::new()
.num_threads(num_threads)
.build()
.context("Failed to create thread pool")?;
SymbolCache::open(&self.cache_path).context("Failed to open symbol cache")?;
let mut conn = crate::cache::open_meta_db(self.cache_path.join("meta.db"))
.context("Failed to open meta.db for the symbol pass")?;
conn.execute_batch("PRAGMA synchronous=NORMAL")
.context("Failed to set synchronous=NORMAL")?;
let content_path = self.cache_path.join("content.bin");
if !content_path.exists() {
log::info!("No content.bin found - index is empty, nothing to process");
self.status.total_files = 0;
self.status.processed_files = 0;
self.write_status()?;
return Ok(());
}
let content_reader =
ContentReader::open(&content_path).context("Failed to open content.bin")?;
let file_rows = cache_mgr
.load_all_file_rows()
.context("Failed to load file rows")?;
let cached_keys =
SymbolCache::load_cached_keys_on(&conn).context("Failed to load cached symbol keys")?;
let total_files = content_reader.file_count();
self.status.total_files = total_files;
log::info!("Found {} indexed files to process", total_files);
if file_rows.is_empty() && total_files > 0 {
log::error!(
"CRITICAL: No file hashes found in file_branches table, but {} files exist in content.bin!",
total_files
);
log::error!("This likely means:");
log::error!(" 1. The main indexer failed to populate file_branches table");
log::error!(" 2. WAL checkpoint didn't flush data before background indexer started");
log::error!(" 3. Database transaction was rolled back");
anyhow::bail!(
"No file hashes available - cannot index symbols. \
This is a database synchronization issue. \
Try running 'rfx index' again or clearing the cache with 'rfx clear'."
);
}
let mut work: Vec<WorkItem> = Vec::new();
let mut cached = 0usize;
let mut no_parser = 0usize;
let mut no_hash = 0usize;
for content_id in 0..total_files as u32 {
let Some(path) = content_reader.get_file_path(content_id) else {
no_hash += 1;
continue;
};
let path_str = path.to_string_lossy();
let path_str = path_str.strip_prefix("./").unwrap_or(&path_str).to_string();
let Some((db_id, hash)) = file_rows.get(&path_str) else {
no_hash += 1;
continue;
};
if cached_keys.contains(&(*db_id, hash.clone())) {
cached += 1;
continue;
}
let language = crate::models::Language::from_path(std::path::Path::new(&path_str));
if !ParserFactory::has_symbol_parser(language) {
no_parser += 1;
continue;
}
work.push(WorkItem {
content_id,
db_id: *db_id,
path: path_str,
hash: hash.clone(),
});
}
drop(cached_keys);
drop(file_rows);
log::info!(
"Symbol pass: {} to parse, {} cached, {} without a symbol parser, {} not in the database",
work.len(),
cached,
no_parser,
no_hash
);
let base_processed = cached + no_parser + no_hash;
self.status.cached_files = cached;
self.status.processed_files = base_processed;
self.status.phase = "parsing".to_string();
self.write_status()?;
let counters = PassCounters::default();
let cancel = AtomicBool::new(false);
let (tx, rx) = std::sync::mpsc::sync_channel::<ParsedFile>(256);
let mut writer_status = self.status.clone();
let cache_path = self.cache_path.clone();
let counters_ref = &counters;
let cancel_ref = &cancel;
let content_ref = &content_reader;
let this = &*self;
let writer_result = std::thread::scope(|scope| {
let writer = scope.spawn(|| {
Self::writer_loop(
rx,
&mut conn,
&cache_path,
&mut writer_status,
base_processed,
counters_ref,
cancel_ref,
)
});
thread_pool.install(|| {
work.par_iter().for_each(|item| {
if cancel_ref.load(Ordering::Relaxed) {
return;
}
let parse_start = Instant::now();
let outcome = this.parse_symbols(content_ref, item.content_id, &item.path);
counters_ref
.parse_ns
.fetch_add(parse_start.elapsed().as_nanos() as u64, Ordering::Relaxed);
match outcome {
Ok(ParseOutcome::Parsed(symbols)) => {
let encode_start = Instant::now();
let encoded = crate::symbol_cache::encode_symbols(&symbols);
counters_ref.encode_ns.fetch_add(
encode_start.elapsed().as_nanos() as u64,
Ordering::Relaxed,
);
counters_ref
.symbols
.fetch_add(symbols.len(), Ordering::Relaxed);
match encoded {
Ok(blob) => {
counters_ref.parsed.fetch_add(1, Ordering::Relaxed);
if tx
.send(ParsedFile {
db_id: item.db_id,
hash: item.hash.clone(),
path: item.path.clone(),
blob,
})
.is_err()
{
cancel_ref.store(true, Ordering::Relaxed);
}
}
Err(e) => {
log::warn!("Failed to encode symbols for {}: {}", item.path, e);
counters_ref.failed.fetch_add(1, Ordering::Relaxed);
}
}
}
Ok(ParseOutcome::SkippedMinified) => {
counters_ref
.skipped_minified
.fetch_add(1, Ordering::Relaxed);
}
Err(e) => {
log::warn!("Failed to parse symbols from {}: {}", item.path, e);
counters_ref.failed.fetch_add(1, Ordering::Relaxed);
}
}
});
});
drop(tx);
writer
.join()
.unwrap_or_else(|_| Err(anyhow::anyhow!("symbol writer thread panicked")))
});
self.status = writer_status;
writer_result?;
log::info!(
"Symbol pass CPU: parse {} ms, encode {} ms across {} threads; {} symbols, {} blob bytes",
counters.parse_ns.load(Ordering::Relaxed) / 1_000_000,
counters.encode_ns.load(Ordering::Relaxed) / 1_000_000,
num_threads,
counters.symbols.load(Ordering::Relaxed),
counters.blob_bytes.load(Ordering::Relaxed)
);
let cancelled =
cancel.load(Ordering::Relaxed) && counters.processed() + base_processed < total_files;
if cancelled {
log::info!(
"Symbol indexing cancelled at {}/{} files (an indexer asked for the database)",
self.status.processed_files,
total_files
);
self.status.state = IndexerState::Cancelled;
self.status.phase = "cancelled".to_string();
self.write_status()?;
return Ok(());
}
self.status.processed_files = total_files;
self.write_status()?;
self.status.phase = "cleanup".to_string();
let _ = self.write_status();
let cleanup_start = Instant::now();
let removed =
SymbolCache::cleanup_stale_on(&conn).context("Failed to cleanup stale symbols")?;
let cleanup_ms = cleanup_start.elapsed().as_millis();
if cleanup_ms > 1000 {
log::warn!(
"cleanup_stale took {}ms for {} removed entries \u{2014} check the index on symbols(file_id)",
cleanup_ms,
removed
);
}
if removed > 0 {
log::info!(
"Cleaned up {} stale symbol entries in {}ms",
removed,
cleanup_ms
);
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
fn writer_loop(
rx: std::sync::mpsc::Receiver<ParsedFile>,
conn: &mut rusqlite::Connection,
cache_path: &Path,
status: &mut IndexingStatus,
base_processed: usize,
counters: &PassCounters,
cancel: &AtomicBool,
) -> Result<()> {
use std::sync::mpsc::RecvTimeoutError;
const BATCH_FILES: usize = 1024;
const BATCH_BYTES: usize = 16 * 1024 * 1024;
const POLL: std::time::Duration = std::time::Duration::from_millis(200);
const STATUS_EVERY: std::time::Duration = std::time::Duration::from_secs(1);
let mut batch: Vec<ParsedFile> = Vec::new();
let mut batch_bytes = 0usize;
let mut last_status = Instant::now();
let refresh = |status: &mut IndexingStatus, phase: &str| {
status.parsed_files = counters.parsed.load(Ordering::Relaxed);
status.failed_files = counters.failed.load(Ordering::Relaxed);
status.skipped_minified = counters.skipped_minified.load(Ordering::Relaxed);
status.write_failed_files = counters.write_failed.load(Ordering::Relaxed);
status.processed_files = base_processed + counters.processed();
status.phase = phase.to_string();
status.current_file = None;
};
loop {
let mut done = false;
match rx.recv_timeout(POLL) {
Ok(item) => {
batch_bytes += item.blob.len();
counters
.blob_bytes
.fetch_add(item.blob.len(), Ordering::Relaxed);
batch.push(item);
}
Err(RecvTimeoutError::Timeout) => {}
Err(RecvTimeoutError::Disconnected) => done = true,
}
if done || batch.len() >= BATCH_FILES || batch_bytes >= BATCH_BYTES {
if !batch.is_empty() {
refresh(status, "writing");
Self::write_status_file(cache_path, status);
Self::commit_batch(conn, &mut batch, counters, status);
batch_bytes = 0;
refresh(status, "parsing");
Self::write_status_file(cache_path, status);
last_status = Instant::now();
}
if done {
break;
}
}
if !cancel.load(Ordering::Relaxed) && Self::cancel_requested(cache_path) {
log::info!("Cancel requested; finishing in-flight files and stopping");
cancel.store(true, Ordering::Relaxed);
}
if last_status.elapsed() >= STATUS_EVERY {
refresh(status, "parsing");
Self::write_status_file(cache_path, status);
last_status = Instant::now();
}
}
refresh(status, "parsing");
Ok(())
}
fn commit_batch(
conn: &mut rusqlite::Connection,
batch: &mut Vec<ParsedFile>,
counters: &PassCounters,
status: &mut IndexingStatus,
) {
let started = Instant::now();
let n = batch.len();
let mut result = Self::try_commit(conn, batch);
if let Err(e) = &result {
log::warn!(
"Symbol batch of {} files failed to commit ({}); retrying once",
n,
e
);
std::thread::sleep(std::time::Duration::from_millis(250));
result = Self::try_commit(conn, batch);
}
match result {
Ok(()) => {
counters.persisted.fetch_add(n, Ordering::Relaxed);
log::info!(
"Symbol batch committed: {} files, {} ms (parsed so far {})",
n,
started.elapsed().as_millis(),
counters.parsed.load(Ordering::Relaxed)
);
}
Err(e) => {
counters.write_failed.fetch_add(n, Ordering::Relaxed);
let first = batch.first().map(|f| f.path.as_str()).unwrap_or("unknown");
status.error = Some(format!(
"{} file(s) parsed but not persisted (first: {}): {:#}",
n, first, e
));
log::error!(
"Failed to write symbol batch of {} file(s) after {} ms, first {}: {:#}",
n,
started.elapsed().as_millis(),
first,
e
);
}
}
batch.clear();
}
fn try_commit(conn: &mut rusqlite::Connection, batch: &[ParsedFile]) -> Result<()> {
let tx = conn
.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)
.context("begin")?;
let now = chrono::Utc::now().timestamp().to_string();
{
let mut stmt = tx.prepare_cached(SymbolCache::INSERT_SYMBOLS_SQL)?;
for item in batch {
stmt.execute(rusqlite::params![item.db_id, item.hash, item.blob, now])?;
}
}
tx.commit().context("commit")?;
Ok(())
}
fn write_status_file(cache_path: &Path, status: &mut IndexingStatus) {
status.updated_at = chrono::Utc::now().to_rfc3339();
let status_path = cache_path.join(STATUS_FILE);
match serde_json::to_string_pretty(status) {
Ok(json) => {
if let Err(e) = std::fs::write(&status_path, json) {
log::warn!("Failed to write status: {}", e);
}
}
Err(e) => log::warn!("Failed to serialize status: {}", e),
}
}
fn parse_symbols(
&self,
content_reader: &ContentReader,
file_id: u32,
path: &str,
) -> Result<ParseOutcome> {
let source = content_reader
.get_file_content(file_id)
.with_context(|| format!("Failed to read file from content.bin: {}", path))?;
let language = crate::models::Language::from_path(std::path::Path::new(path));
if crate::parsers::is_minified(source) {
return Ok(ParseOutcome::SkippedMinified);
}
let symbols = ParserFactory::parse(path, source, language)
.with_context(|| format!("Failed to parse symbols from: {}", path))?;
Ok(ParseOutcome::Parsed(symbols))
}
}
struct WorkItem {
content_id: u32,
db_id: i64,
path: String,
hash: String,
}
struct ParsedFile {
db_id: i64,
hash: String,
path: String,
blob: Vec<u8>,
}
#[derive(Default)]
struct PassCounters {
parsed: AtomicUsize,
failed: AtomicUsize,
skipped_minified: AtomicUsize,
persisted: AtomicUsize,
write_failed: AtomicUsize,
parse_ns: std::sync::atomic::AtomicU64,
encode_ns: std::sync::atomic::AtomicU64,
symbols: AtomicUsize,
blob_bytes: AtomicUsize,
}
impl PassCounters {
fn processed(&self) -> usize {
self.parsed.load(Ordering::Relaxed)
+ self.failed.load(Ordering::Relaxed)
+ self.skipped_minified.load(Ordering::Relaxed)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::cache::CacheManager;
use tempfile::TempDir;
#[test]
fn test_indexer_lock() {
let temp = TempDir::new().unwrap();
let cache_mgr = CacheManager::new(temp.path());
cache_mgr.init().unwrap();
assert!(!BackgroundIndexer::is_running(cache_mgr.path()));
let indexer = BackgroundIndexer::new(temp.path()).unwrap();
let _lock = indexer.acquire_lock().unwrap();
assert!(BackgroundIndexer::is_running(cache_mgr.path()));
indexer.release_lock().unwrap();
assert!(!BackgroundIndexer::is_running(cache_mgr.path()));
}
#[test]
fn test_indexer_lock_prevents_concurrent() {
let temp = TempDir::new().unwrap();
let cache_mgr = CacheManager::new(temp.path());
cache_mgr.init().unwrap();
let indexer1 = BackgroundIndexer::new(temp.path()).unwrap();
let _lock1 = indexer1.acquire_lock().unwrap();
let indexer2 = BackgroundIndexer::new(temp.path()).unwrap();
let result = indexer2.acquire_lock();
assert!(result.is_err());
assert!(
result
.unwrap_err()
.to_string()
.contains("already in progress")
);
}
#[test]
fn test_indexer_status_write() {
let temp = TempDir::new().unwrap();
let cache_mgr = CacheManager::new(temp.path());
cache_mgr.init().unwrap();
let mut indexer = BackgroundIndexer::new(temp.path()).unwrap();
indexer.status.total_files = 100;
indexer.status.processed_files = 50;
indexer.write_status().unwrap();
let status = BackgroundIndexer::get_status(cache_mgr.path()).unwrap();
assert!(status.is_some());
let status = status.unwrap();
assert_eq!(status.total_files, 100);
assert_eq!(status.processed_files, 50);
assert_eq!(status.state, IndexerState::Running);
}
#[test]
fn test_indexer_status_read_nonexistent() {
let temp = TempDir::new().unwrap();
let cache_mgr = CacheManager::new(temp.path());
cache_mgr.init().unwrap();
let status = BackgroundIndexer::get_status(cache_mgr.path()).unwrap();
assert!(status.is_none());
}
#[test]
fn test_indexer_run_empty_index() {
let temp = TempDir::new().unwrap();
let cache_mgr = CacheManager::new(temp.path());
cache_mgr.init().unwrap();
let mut indexer = BackgroundIndexer::new(temp.path()).unwrap();
let result = indexer.run();
assert!(result.is_ok());
assert_eq!(indexer.status.state, IndexerState::Completed);
assert_eq!(indexer.status.processed_files, 0);
assert_eq!(indexer.status.total_files, 0);
}
#[test]
fn test_stale_lock_detection() {
use filetime::{FileTime, set_file_mtime};
let temp = TempDir::new().unwrap();
let cache_mgr = CacheManager::new(temp.path());
cache_mgr.init().unwrap();
let lock_path = cache_mgr.path().join(LOCK_FILE);
std::fs::write(&lock_path, "12345").unwrap();
assert!(!is_lock_stale(&lock_path), "fresh lock should not be stale");
let two_hours_ago = std::time::SystemTime::now() - std::time::Duration::from_secs(2 * 3600);
set_file_mtime(&lock_path, FileTime::from_system_time(two_hours_ago)).unwrap();
assert!(is_lock_stale(&lock_path), "2-hour-old lock should be stale");
std::fs::remove_file(&lock_path).unwrap();
assert!(
!is_lock_stale(&lock_path),
"missing lock should not be stale"
);
}
#[test]
fn test_is_running_cleans_stale_lock() {
use filetime::{FileTime, set_file_mtime};
let temp = TempDir::new().unwrap();
let cache_mgr = CacheManager::new(temp.path());
cache_mgr.init().unwrap();
let lock_path = cache_mgr.path().join(LOCK_FILE);
std::fs::write(&lock_path, "99999999").unwrap();
let two_hours_ago = std::time::SystemTime::now() - std::time::Duration::from_secs(2 * 3600);
set_file_mtime(&lock_path, FileTime::from_system_time(two_hours_ago)).unwrap();
assert!(
lock_path.exists(),
"lock file should exist before is_running()"
);
assert!(!BackgroundIndexer::is_running(cache_mgr.path()));
assert!(
!lock_path.exists(),
"stale lock file should be removed by is_running()"
);
}
#[test]
fn test_acquire_lock_cleans_stale_lock() {
use filetime::{FileTime, set_file_mtime};
let temp = TempDir::new().unwrap();
let cache_mgr = CacheManager::new(temp.path());
cache_mgr.init().unwrap();
let lock_path = cache_mgr.path().join(LOCK_FILE);
std::fs::write(&lock_path, "99999999").unwrap();
let two_hours_ago = std::time::SystemTime::now() - std::time::Duration::from_secs(2 * 3600);
set_file_mtime(&lock_path, FileTime::from_system_time(two_hours_ago)).unwrap();
let indexer = BackgroundIndexer::new(temp.path()).unwrap();
let _lock = indexer
.acquire_lock()
.expect("stale lock should not block acquire_lock");
assert!(lock_path.exists(), "new lock file should be created");
}
fn workspace(files: usize) -> TempDir {
let temp = TempDir::new().unwrap();
let src = temp.path().join("src");
std::fs::create_dir_all(&src).unwrap();
for i in 0..files {
std::fs::write(
src.join(format!("m{i}.rs")),
format!(
"pub fn func_{i}(x: u32) -> u32 {{\n let local_{i} = x + {i};\n local_{i}\n}}\n\npub struct S{i} {{\n pub a: u32,\n}}\n"
),
)
.unwrap();
}
std::fs::write(temp.path().join("README.md"), "# no symbols here\n").unwrap();
std::fs::write(temp.path().join("notes.txt"), "plain text\n").unwrap();
temp
}
fn index_workspace(root: &Path) {
let cache = CacheManager::new(root);
crate::indexer::Indexer::new(cache, crate::models::IndexConfig::default())
.index(root, false)
.unwrap();
}
fn symbol_rows(root: &Path) -> i64 {
let conn = crate::cache::open_meta_db(root.join(".reflex").join("meta.db")).unwrap();
conn.query_row("SELECT COUNT(*) FROM symbols", [], |r| r.get(0))
.unwrap()
}
#[test]
fn full_run_parses_every_code_file_then_reports_them_cached() {
let temp = workspace(40);
index_workspace(temp.path());
let mut first = BackgroundIndexer::new(temp.path()).unwrap();
first.run().unwrap();
assert_eq!(first.status.state, IndexerState::Completed);
assert_eq!(
first.status.total_files, 42,
"40 .rs + README.md + notes.txt"
);
assert_eq!(first.status.processed_files, 42);
assert_eq!(first.status.parsed_files, 40);
assert_eq!(first.status.cached_files, 0);
assert_eq!(first.status.failed_files, 0);
assert_eq!(first.status.write_failed_files, 0);
assert!(first.status.error.is_none(), "{:?}", first.status.error);
assert_eq!(symbol_rows(temp.path()), 40);
let mut second = BackgroundIndexer::new(temp.path()).unwrap();
second.run().unwrap();
assert_eq!(second.status.state, IndexerState::Completed);
assert_eq!(second.status.cached_files, 40);
assert_eq!(second.status.parsed_files, 0);
assert_eq!(second.status.processed_files, 42);
let symbol_cache = SymbolCache::open(temp.path().join(".reflex").as_path()).unwrap();
let conn = crate::cache::open_meta_db(temp.path().join(".reflex").join("meta.db")).unwrap();
let hash: String = conn
.query_row(
"SELECT fb.hash FROM file_branches fb JOIN files f ON f.id = fb.file_id WHERE f.path = 'src/m7.rs'",
[],
|r| r.get(0),
)
.unwrap();
let cached = symbol_cache.get("src/m7.rs", &hash).unwrap().unwrap();
let names: Vec<_> = cached.iter().filter_map(|s| s.symbol.as_deref()).collect();
assert!(names.contains(&"func_7"), "{names:?}");
assert!(names.contains(&"S7"), "{names:?}");
assert!(cached.iter().all(|s| s.path == "src/m7.rs"));
}
#[test]
fn cancel_request_stops_the_pass_and_keeps_what_was_parsed() {
let temp = workspace(1500);
index_workspace(temp.path());
let cache_path = temp.path().join(".reflex");
let canceller = {
let cache_path = cache_path.clone();
std::thread::spawn(move || {
for _ in 0..500 {
if let Ok(Some(st)) = BackgroundIndexer::get_status(&cache_path)
&& st.phase == "parsing"
{
break;
}
std::thread::sleep(std::time::Duration::from_millis(2));
}
BackgroundIndexer::request_cancel(&cache_path).unwrap();
})
};
let mut indexer = BackgroundIndexer::new(temp.path()).unwrap();
indexer.run().unwrap();
canceller.join().unwrap();
let st = &indexer.status;
assert!(
matches!(st.state, IndexerState::Cancelled | IndexerState::Completed),
"{:?}",
st.state
);
assert_eq!(symbol_rows(temp.path()), st.parsed_files as i64);
assert_eq!(st.write_failed_files, 0);
if st.state == IndexerState::Cancelled {
assert!(st.processed_files < st.total_files, "{st:?}");
assert_eq!(st.phase, "cancelled");
}
assert!(!cache_path.join(LOCK_FILE).exists());
BackgroundIndexer::clear_cancel(&cache_path);
let mut again = BackgroundIndexer::new(temp.path()).unwrap();
again.run().unwrap();
assert_eq!(again.status.state, IndexerState::Completed);
assert_eq!(symbol_rows(temp.path()), 1500);
assert_eq!(again.status.cached_files + again.status.parsed_files, 1500);
}
#[test]
fn write_failure_is_counted_apart_from_parse_failures() {
let temp = workspace(5);
index_workspace(temp.path());
let cache_path = temp.path().join(".reflex");
let conn = crate::cache::open_meta_db(cache_path.join("meta.db")).unwrap();
SymbolCache::ensure_schema(&conn).unwrap();
conn.execute_batch(
"CREATE TRIGGER reject_symbol_writes BEFORE INSERT ON symbols
BEGIN SELECT RAISE(ABORT, 'injected write failure'); END;",
)
.unwrap();
drop(conn);
let mut indexer = BackgroundIndexer::new(temp.path()).unwrap();
let result = indexer.run();
assert!(result.is_ok(), "{result:?}");
let st = &indexer.status;
assert_eq!(st.state, IndexerState::Completed);
assert_eq!(st.parsed_files, 5, "parsing succeeded");
assert_eq!(st.failed_files, 0, "a write failure is not a parse failure");
assert_eq!(st.write_failed_files, 5, "{st:?}");
let err = st.error.as_deref().unwrap_or("");
assert!(err.contains("5 file(s) parsed but not persisted"), "{err}");
assert!(err.contains("src/m"), "{err}");
}
}