mod chunker;
mod imports;
mod languages;
pub use chunker::{chunk_file, chunk_plain_text, ChunkConfig};
pub use imports::extract_imports;
pub use languages::{get_language, get_non_code_language};
use crate::db::{CodeChunk, Database};
use anyhow::Result;
use ignore::WalkBuilder;
use indicatif::{ProgressBar, ProgressStyle};
use sha2::{Digest, Sha256};
use std::path::Path;
pub struct Indexer<'a> {
db: &'a Database,
config: IndexConfig,
}
#[derive(Clone)]
pub struct IndexConfig {
pub chunk_config: ChunkConfig,
pub force_reindex: bool,
}
impl Default for IndexConfig {
fn default() -> Self {
Self {
chunk_config: ChunkConfig::default(),
force_reindex: false,
}
}
}
impl<'a> Indexer<'a> {
pub fn new(db: &'a Database, config: IndexConfig) -> Self {
Self { db, config }
}
pub fn index_directory(&mut self, path: &Path) -> Result<(IndexResult, Vec<CodeChunk>)> {
let mut result = IndexResult::default();
let mut new_chunks: Vec<CodeChunk> = Vec::new();
let files: Vec<_> = WalkBuilder::new(path)
.hidden(false)
.git_ignore(true)
.git_global(true)
.git_exclude(true)
.build()
.filter_map(|e| e.ok())
.filter(|e| e.file_type().map(|ft| ft.is_file()).unwrap_or(false))
.filter(|e| {
let path = e.path();
let ext = path.extension().and_then(|s| s.to_str()).unwrap_or("");
let filename = path.file_name().and_then(|s| s.to_str()).unwrap_or("");
get_language(ext).is_some() || get_non_code_language(filename, ext).is_some()
})
.collect();
let disk_files: std::collections::HashSet<String> = files
.iter()
.map(|e| {
e.path()
.strip_prefix(&self.db.project_root)
.unwrap_or(e.path())
.to_string_lossy()
.to_string()
})
.collect();
let tracked_files = self.db.get_all_tracked_files()?;
for tracked in &tracked_files {
if !disk_files.contains(tracked.as_str()) {
let old_ids = self.db.get_chunk_ids_by_file(tracked)?;
result.old_chunk_ids.extend(old_ids);
self.db.delete_chunks_by_file(tracked)?;
self.db.delete_edges_by_file(tracked)?;
self.db.delete_file_hash(tracked)?;
result.files_deleted += 1;
}
}
let progress = ProgressBar::new(files.len() as u64);
progress.set_style(
ProgressStyle::default_bar()
.template(
"{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {pos}/{len} {msg}",
)
.unwrap()
.progress_chars("#>-"),
);
for entry in files {
let file_path = entry.path();
progress.set_message(format!("{}", file_path.display()));
match self.process_file(file_path, &mut result) {
Ok(Some(processed)) => {
result.old_chunk_ids.extend(processed.old_chunk_ids);
for chunk in &processed.chunks {
result.new_chunk_ids.push(chunk.id.clone());
}
for chunk in &processed.chunks {
self.db.insert_chunk(chunk)?;
}
new_chunks.extend(processed.chunks);
for edge in &processed.edges {
self.db.insert_edge(edge)?;
}
result.edges_created += processed.edges.len();
}
Ok(None) => {
result.skipped += 1;
}
Err(e) => {
result
.errors
.push(format!("{}: {}", file_path.display(), e));
}
}
progress.inc(1);
}
progress.finish_with_message("Done!");
result.edges_total = self.db.get_total_edge_count()?;
result.edges_resolved = self.db.resolve_edge_targets()?;
Ok((result, new_chunks))
}
fn process_file(&self, path: &Path, result: &mut IndexResult) -> Result<Option<ProcessedFile>> {
let relative_path = path
.strip_prefix(&self.db.project_root)
.unwrap_or(path)
.to_string_lossy()
.to_string();
let content = std::fs::read_to_string(path)?;
let content_hash = compute_hash(&content);
if !self.config.force_reindex {
if let Some(stored_hash) = self.db.get_file_hash(&relative_path)? {
if stored_hash == content_hash {
return Ok(None);
}
}
}
let old_chunk_ids = self.db.get_chunk_ids_by_file(&relative_path)?;
self.db.delete_chunks_by_file(&relative_path)?;
self.db.delete_edges_by_file(&relative_path)?;
let ext = path.extension().and_then(|s| s.to_str()).unwrap_or("");
let filename = path.file_name().and_then(|s| s.to_str()).unwrap_or("");
let (chunks, edges) = if let Some(lang) = get_language(ext) {
let (chunks, tree) =
chunk_file(&content, &relative_path, lang, &self.config.chunk_config)?;
let edges = extract_imports(&tree, &content, lang, &relative_path);
(chunks, edges)
} else if let Some(lang_label) = get_non_code_language(filename, ext) {
let chunks = chunk_plain_text(
&content,
&relative_path,
lang_label,
&self.config.chunk_config,
)?;
(chunks, Vec::new())
} else {
return Ok(None);
};
self.db.set_file_hash(&relative_path, &content_hash)?;
result.files_indexed += 1;
result.chunks_created += chunks.len();
Ok(Some(ProcessedFile {
chunks,
edges,
old_chunk_ids,
}))
}
}
fn compute_hash(content: &str) -> String {
let mut hasher = Sha256::new();
hasher.update(content.as_bytes());
hex::encode(hasher.finalize())
}
#[derive(Default)]
pub struct IndexResult {
pub files_indexed: usize,
pub chunks_created: usize,
pub edges_created: usize,
pub edges_resolved: usize,
pub edges_total: usize,
pub skipped: usize,
pub files_deleted: usize,
pub old_chunk_ids: Vec<String>,
pub new_chunk_ids: Vec<String>,
pub errors: Vec<String>,
}
pub struct ProcessedFile {
pub chunks: Vec<CodeChunk>,
pub edges: Vec<crate::db::SymbolEdge>,
pub old_chunk_ids: Vec<String>,
}