minni 0.1.0

Local memory, task, and codebase indexing tool for AI agents
Documentation
mod chunker;
mod imports;
mod languages;

pub use chunker::{chunk_file, chunk_plain_text, ChunkConfig};
pub use imports::extract_imports;
pub use languages::{get_language, get_non_code_language};

use crate::db::{CodeChunk, Database};
use anyhow::Result;
use ignore::WalkBuilder;
use indicatif::{ProgressBar, ProgressStyle};
use sha2::{Digest, Sha256};
use std::path::Path;

/// Codebase indexer.
pub struct Indexer<'a> {
    db: &'a Database,
    config: IndexConfig,
}

/// Indexing options.
#[derive(Clone)]
pub struct IndexConfig {
    pub chunk_config: ChunkConfig,
    pub force_reindex: bool,
}

impl Default for IndexConfig {
    fn default() -> Self {
        Self {
            chunk_config: ChunkConfig::default(),
            force_reindex: false,
        }
    }
}

impl<'a> Indexer<'a> {
    pub fn new(db: &'a Database, config: IndexConfig) -> Self {
        Self { db, config }
    }

    pub fn index_directory(&mut self, path: &Path) -> Result<(IndexResult, Vec<CodeChunk>)> {
        let mut result = IndexResult::default();
        let mut new_chunks: Vec<CodeChunk> = Vec::new();

        // Collect all files to index
        let files: Vec<_> = WalkBuilder::new(path)
            .hidden(false)
            .git_ignore(true)
            .git_global(true)
            .git_exclude(true)
            .build()
            .filter_map(|e| e.ok())
            .filter(|e| e.file_type().map(|ft| ft.is_file()).unwrap_or(false))
            .filter(|e| {
                let path = e.path();
                let ext = path.extension().and_then(|s| s.to_str()).unwrap_or("");
                let filename = path.file_name().and_then(|s| s.to_str()).unwrap_or("");
                get_language(ext).is_some() || get_non_code_language(filename, ext).is_some()
            })
            .collect();

        // Build set of current on-disk file paths (relative)
        let disk_files: std::collections::HashSet<String> = files
            .iter()
            .map(|e| {
                e.path()
                    .strip_prefix(&self.db.project_root)
                    .unwrap_or(e.path())
                    .to_string_lossy()
                    .to_string()
            })
            .collect();

        // Detect deleted files: tracked in DB but no longer on disk
        let tracked_files = self.db.get_all_tracked_files()?;
        for tracked in &tracked_files {
            if !disk_files.contains(tracked.as_str()) {
                // Collect old chunk IDs before deleting
                let old_ids = self.db.get_chunk_ids_by_file(tracked)?;
                result.old_chunk_ids.extend(old_ids);

                self.db.delete_chunks_by_file(tracked)?;
                self.db.delete_edges_by_file(tracked)?;
                self.db.delete_file_hash(tracked)?;
                result.files_deleted += 1;
            }
        }

        let progress = ProgressBar::new(files.len() as u64);
        progress.set_style(
            ProgressStyle::default_bar()
                .template(
                    "{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {pos}/{len} {msg}",
                )
                .unwrap()
                .progress_chars("#>-"),
        );

        for entry in files {
            let file_path = entry.path();
            progress.set_message(format!("{}", file_path.display()));

            match self.process_file(file_path, &mut result) {
                Ok(Some(processed)) => {
                    // Accumulate old chunk IDs for BM25 deletion
                    result.old_chunk_ids.extend(processed.old_chunk_ids);
                    // Accumulate new chunk IDs and chunks
                    for chunk in &processed.chunks {
                        result.new_chunk_ids.push(chunk.id.clone());
                    }
                    // Store chunks in database
                    for chunk in &processed.chunks {
                        self.db.insert_chunk(chunk)?;
                    }
                    new_chunks.extend(processed.chunks);
                    // Store import edges
                    for edge in &processed.edges {
                        self.db.insert_edge(edge)?;
                    }
                    result.edges_created += processed.edges.len();
                }
                Ok(None) => {
                    result.skipped += 1;
                }
                Err(e) => {
                    result
                        .errors
                        .push(format!("{}: {}", file_path.display(), e));
                }
            }

            progress.inc(1);
        }

        progress.finish_with_message("Done!");

        // Resolve target_file for all unresolved edges
        result.edges_total = self.db.get_total_edge_count()?;
        result.edges_resolved = self.db.resolve_edge_targets()?;

        Ok((result, new_chunks))
    }

    fn process_file(&self, path: &Path, result: &mut IndexResult) -> Result<Option<ProcessedFile>> {
        let relative_path = path
            .strip_prefix(&self.db.project_root)
            .unwrap_or(path)
            .to_string_lossy()
            .to_string();

        // Read file content
        let content = std::fs::read_to_string(path)?;
        let content_hash = compute_hash(&content);

        // Check if file needs reindexing
        if !self.config.force_reindex {
            if let Some(stored_hash) = self.db.get_file_hash(&relative_path)? {
                if stored_hash == content_hash {
                    return Ok(None);
                }
            }
        }

        // Collect old chunk IDs before deleting (for BM25 cleanup)
        let old_chunk_ids = self.db.get_chunk_ids_by_file(&relative_path)?;

        // Delete old chunks and edges for this file
        self.db.delete_chunks_by_file(&relative_path)?;
        self.db.delete_edges_by_file(&relative_path)?;

        // Get language for this file and chunk accordingly
        let ext = path.extension().and_then(|s| s.to_str()).unwrap_or("");
        let filename = path.file_name().and_then(|s| s.to_str()).unwrap_or("");

        let (chunks, edges) = if let Some(lang) = get_language(ext) {
            // Code file: use tree-sitter chunking; also extract import edges
            let (chunks, tree) =
                chunk_file(&content, &relative_path, lang, &self.config.chunk_config)?;
            let edges = extract_imports(&tree, &content, lang, &relative_path);
            (chunks, edges)
        } else if let Some(lang_label) = get_non_code_language(filename, ext) {
            // Non-code file: use plain-text chunking; no import edges
            let chunks = chunk_plain_text(
                &content,
                &relative_path,
                lang_label,
                &self.config.chunk_config,
            )?;
            (chunks, Vec::new())
        } else {
            return Ok(None);
        };

        // Update file hash
        self.db.set_file_hash(&relative_path, &content_hash)?;

        result.files_indexed += 1;
        result.chunks_created += chunks.len();

        Ok(Some(ProcessedFile {
            chunks,
            edges,
            old_chunk_ids,
        }))
    }
}

fn compute_hash(content: &str) -> String {
    let mut hasher = Sha256::new();
    hasher.update(content.as_bytes());
    hex::encode(hasher.finalize())
}

/// Indexing summary.
#[derive(Default)]
pub struct IndexResult {
    pub files_indexed: usize,
    pub chunks_created: usize,
    pub edges_created: usize,
    pub edges_resolved: usize,
    pub edges_total: usize,
    pub skipped: usize,
    pub files_deleted: usize,
    pub old_chunk_ids: Vec<String>,
    pub new_chunk_ids: Vec<String>,
    pub errors: Vec<String>,
}

/// File processing output.
pub struct ProcessedFile {
    pub chunks: Vec<CodeChunk>,
    pub edges: Vec<crate::db::SymbolEdge>,
    pub old_chunk_ids: Vec<String>,
}