ripvec_core/encoder/ripvec/

index.rs

//! `RipvecIndex` orchestrator and PageRank-layered ranking.
//!
//! Port of `~/src/semble/src/semble/index/index.py:RipvecIndex`. Owns
//! the corpus state (chunks, file mapping, language mapping, BM25,
//! dense embeddings, encoder) and dispatches search by mode.
//!
//! ## Port-plus-ripvec scope
//!
//! Per `docs/PLAN.md`, after the ripvec engine's own `rerank_topk` runs, ripvec's
//! [`boost_with_pagerank`](crate::hybrid::boost_with_pagerank) is
//! applied as a final ranking layer. The PageRank lookup is built from
//! the repo graph and stored alongside the corpus when one is provided
//! at construction; the layer no-ops when no graph is present.

use std::collections::HashMap;
use std::path::{Path, PathBuf};

use crate::chunk::CodeChunk;
use crate::embed::SearchConfig;
use crate::encoder::VectorEncoder;
use crate::encoder::ripvec::bm25::{Bm25Index, search_bm25};
use crate::encoder::ripvec::dense::StaticEncoder;
use crate::encoder::ripvec::hybrid::{search_hybrid, search_semantic};
use crate::encoder::ripvec::manifest::{Diff, FileEntry, Manifest, diff_against_walk};
use crate::hybrid::SearchMode;
use crate::profile::Profiler;
use crate::walk::{WalkOptions, collect_files_with_options};

/// Combined orchestrator for the ripvec retrieval pipeline.
///
/// Constructed via [`RipvecIndex::from_root`] which walks files,
/// chunks them with ripvec's chunker, embeds with the static encoder,
/// and builds the BM25 index.
pub struct RipvecIndex {
    chunks: Vec<CodeChunk>,
    /// Row-major contiguous embedding matrix; row `i` is the
    /// L2-normalized embedding of chunk `i`. Held as `Array2<f32>` so
    /// cosine queries (dot product over normalized rows) dispatch to
    /// BLAS `sgemv` via ndarray's `cpu-accelerate` feature instead of
    /// pointer-chasing through `Vec<Vec<f32>>`. The change is a
    /// ~150x theoretical lift on per-query dense scoring at 1M chunks
    /// (memory-bandwidth-bound).
    embeddings: ndarray::Array2<f32>,
    bm25: Bm25Index,
    /// Shared by `Arc` so [`Self::apply_diff`] can produce a new index
    /// that reuses the same loaded model without cloning the ~32 MB
    /// embedding table. The encoder is immutable after construction.
    encoder: std::sync::Arc<StaticEncoder>,
    file_mapping: HashMap<String, Vec<usize>>,
    language_mapping: HashMap<String, Vec<usize>>,
    pagerank_lookup: Option<std::sync::Arc<HashMap<String, f32>>>,
    pagerank_alpha: f32,
    corpus_class: CorpusClass,
    /// Canonical root the index was built against. Used by
    /// [`RipvecIndex::diff_against_filesystem`] to walk the same tree
    /// for reconciliation.
    root: PathBuf,
    /// Walk filters captured at build time so reconciliation honors the
    /// same `.gitignore`, extension whitelist, ignore-pattern set as
    /// the original index.
    walk_options: WalkOptions,
    /// Per-file fingerprint table (mtime, size, inode, blake3) for
    /// online change detection. Built during [`Self::from_root`] and
    /// queried by [`Self::diff_against_filesystem`]. See
    /// [`crate::encoder::ripvec::manifest`] for the algorithm.
    manifest: Manifest,
}

/// Index-time classification of the corpus by file mix.
///
/// Drives the corpus-aware rerank gate: docs and mixed corpora get
/// the L-12 cross-encoder fired (when the query is NL-shaped); pure
/// code corpora skip it because the ms-marco-trained model is
/// out-of-domain for code regardless of impl quality.
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum CorpusClass {
    /// Less than 30% of chunks are in prose files. Pure or near-pure
    /// code corpora — rerank skipped.
    Code,
    /// Between 30% and 70% prose chunks. Mixed corpora — rerank fires
    /// on NL queries to recover the prose-dominant relevance signal.
    Mixed,
    /// At least 70% prose chunks. Documentation, book sets, knowledge
    /// bases — rerank fires by default.
    Docs,
}

impl CorpusClass {
    /// Classify a chunk set by the fraction of chunks from prose files.
    /// Empty input is classified as `Code` (degenerate but defined).
    #[must_use]
    pub fn classify(chunks: &[CodeChunk]) -> Self {
        if chunks.is_empty() {
            return Self::Code;
        }
        let prose = chunks
            .iter()
            .filter(|c| crate::encoder::ripvec::ranking::is_prose_path(&c.file_path))
            .count();
        #[expect(
            clippy::cast_precision_loss,
            reason = "chunk count never exceeds f32 mantissa precision in practice"
        )]
        let frac = prose as f32 / chunks.len() as f32;
        if frac >= 0.7 {
            Self::Docs
        } else if frac >= 0.3 {
            Self::Mixed
        } else {
            Self::Code
        }
    }

    /// Whether the cross-encoder rerank should run on this corpus for
    /// a non-symbol NL query. Pure code corpora skip rerank; mixed
    /// and docs corpora enable it.
    #[must_use]
    pub fn rerank_eligible(self) -> bool {
        matches!(self, Self::Mixed | Self::Docs)
    }
}

impl RipvecIndex {
    /// Build a [`RipvecIndex`] by walking `root` and indexing every
    /// supported file. Uses `encoder.embed_root` (ripvec's chunker +
    /// model2vec encode) and builds a fresh BM25 index over the
    /// resulting chunks.
    ///
    /// `pagerank_lookup` is the optional structural-prior map (file
    /// path → normalized PageRank) used by the final ranking layer;
    /// pass `None` to disable. `pagerank_alpha` is the corresponding
    /// boost strength.
    ///
    /// # Errors
    ///
    /// Returns the underlying error if `embed_root` fails.
    pub fn from_root(
        root: &Path,
        encoder: StaticEncoder,
        cfg: &SearchConfig,
        profiler: &Profiler,
        pagerank_lookup: Option<HashMap<String, f32>>,
        pagerank_alpha: f32,
    ) -> crate::Result<Self> {
        // Wrap once at construction. The per-query `apply_pagerank_layer`
        // path clones the Arc (pointer bump), not the HashMap (10K+ String
        // allocs on a 1M-chunk corpus).
        let pagerank_lookup = pagerank_lookup.map(std::sync::Arc::new);
        let (chunks, embeddings_vec) = encoder.embed_root(root, cfg, profiler)?;
        // Convert Vec<Vec<f32>> -> Array2<f32> at the boundary. The
        // upstream embed_root produces ragged-friendly Vec<Vec<>>; we
        // pack into one contiguous row-major buffer so BLAS sgemv can
        // do per-query cosine in one call. Cost is a single sequential
        // memcpy pass (~1 GB at memory bandwidth = ~5 ms on a 1M-chunk
        // corpus) — negligible against the 60 s build phase.
        let hidden_dim = embeddings_vec.first().map_or(0, std::vec::Vec::len);
        let n_chunks = embeddings_vec.len();
        let mut flat: Vec<f32> = Vec::with_capacity(n_chunks * hidden_dim);
        for row in embeddings_vec {
            debug_assert_eq!(
                row.len(),
                hidden_dim,
                "ragged embeddings: row of {} vs expected {hidden_dim}",
                row.len()
            );
            flat.extend(row);
        }
        let embeddings = ndarray::Array2::from_shape_vec((n_chunks, hidden_dim), flat)
            .map_err(|e| crate::Error::Other(anyhow::anyhow!("embeddings reshape: {e}")))?;
        let bm25 = {
            let _g = profiler.phase("bm25_build");
            Bm25Index::build(&chunks)
        };
        let (file_mapping, language_mapping) = {
            let _g = profiler.phase("mappings");
            build_mappings(&chunks)
        };
        let corpus_class = CorpusClass::classify(&chunks);
        // Capture walk options for future reconciles, and populate the
        // manifest from the same file set the indexer consumed. We
        // re-walk + re-read here because `embed_root` doesn't surface
        // the per-file bytes back to us; the redundant read is paid
        // once at index build time, not per query. On reconcile we
        // only re-read files whose stat tuple changed.
        let walk_options = cfg.walk_options();
        let root_buf = root.to_path_buf();
        let manifest = {
            let _g = profiler.phase("manifest_build");
            build_manifest(&root_buf, &walk_options)
        };
        Ok(Self {
            chunks,
            embeddings,
            bm25,
            encoder: std::sync::Arc::new(encoder),
            file_mapping,
            language_mapping,
            pagerank_lookup,
            pagerank_alpha,
            corpus_class,
            root: root_buf,
            walk_options,
            manifest,
        })
    }

    /// Build a new index by incrementally applying `diff` against
    /// `self`.
    ///
    /// **The selective-rebuild path that v3.1.0 punted on.** Re-embeds
    /// only the dirty + new files, splices them into the existing
    /// chunks/embeddings, drops deleted files' chunks, rebuilds BM25
    /// and the per-file/per-language mappings from the new chunk set,
    /// reclassifies the corpus, and refreshes the manifest entries
    /// for the affected files.
    ///
    /// # Cost shape
    ///
    /// Roughly `O(|diff.dirty| + |diff.new|)` chunk + embed work plus
    /// `O(|self.chunks|)` BM25 rebuild. On a 5000-chunk corpus with
    /// one file changed: ~5-10 ms (embed one file) + ~50 ms (BM25
    /// rebuild) = ~60 ms — vs. ~270 ms-1 s for a full
    /// [`Self::from_root`] rebuild. The full-build cost is paid only
    /// at cold start.
    ///
    /// # BM25
    ///
    /// BM25 is rebuilt from scratch over the new chunks vec rather
    /// than incrementally updated. Inverted-postings incremental
    /// update is correct but adds significant code; full rebuild at
    /// our chunk counts is fast enough that the simpler path wins.
    ///
    /// # Errors
    ///
    /// Returns the underlying error if [`StaticEncoder::embed_paths`]
    /// fails or if the embedding matrix shape is invalid.
    pub fn apply_diff(&self, diff: &Diff, profiler: &Profiler) -> crate::Result<Self> {
        use std::collections::HashSet;

        // 1. Identify which existing chunk indices to drop. `file_mapping`
        //    keys are the rel_paths the chunker wrote. Manifest paths are
        //    absolute. Map manifest paths to rel_paths by stripping
        //    `self.root` (the same operation `chunk_one_file` performs).
        let rel_path_for = |p: &Path| -> String {
            p.strip_prefix(&self.root)
                .unwrap_or(p)
                .display()
                .to_string()
        };
        let mut removed_indices: HashSet<usize> = HashSet::new();
        for path in diff
            .deleted
            .iter()
            .chain(diff.dirty.iter())
            .chain(diff.new.iter())
        {
            let rel = rel_path_for(path);
            if let Some(indices) = self.file_mapping.get(&rel) {
                removed_indices.extend(indices.iter().copied());
            }
        }

        // 2. Build the kept chunks + embeddings from `self`. Cloning the
        //    embedding rows is one allocation per kept chunk; for a 5k-
        //    chunk corpus that's a single sequential pass over 5 MB.
        let mut kept_chunks: Vec<CodeChunk> = Vec::with_capacity(self.chunks.len());
        let mut kept_emb_rows: Vec<Vec<f32>> = Vec::with_capacity(self.chunks.len());
        for (i, chunk) in self.chunks.iter().enumerate() {
            if removed_indices.contains(&i) {
                continue;
            }
            kept_chunks.push(chunk.clone());
            kept_emb_rows.push(self.embeddings.row(i).to_vec());
        }

        // 3. Embed the dirty + new files. (Dirty files were already
        //    dropped from `kept_chunks` above; their new chunks come in
        //    here as fresh entries.)
        let mut to_embed: Vec<std::path::PathBuf> = Vec::new();
        to_embed.extend(diff.new.iter().cloned());
        to_embed.extend(diff.dirty.iter().cloned());
        let (new_chunks, new_embs) = if to_embed.is_empty() {
            (Vec::new(), Vec::new())
        } else {
            let _g = profiler.phase("apply_diff_embed");
            self.encoder.embed_paths(&self.root, &to_embed, profiler)?
        };
        kept_chunks.extend(new_chunks);
        kept_emb_rows.extend(new_embs);

        // 4. Re-pack embeddings into a contiguous Array2 so BLAS sgemv
        //    still works at query time.
        let n = kept_emb_rows.len();
        let hidden_dim = kept_emb_rows
            .first()
            .map_or(self.embeddings.ncols(), Vec::len);
        let mut flat: Vec<f32> = Vec::with_capacity(n * hidden_dim);
        for row in kept_emb_rows {
            flat.extend(row);
        }
        let embeddings = if n == 0 {
            ndarray::Array2::<f32>::zeros((0, hidden_dim))
        } else {
            ndarray::Array2::from_shape_vec((n, hidden_dim), flat).map_err(|e| {
                crate::Error::Other(anyhow::anyhow!("apply_diff embeddings reshape: {e}"))
            })?
        };

        // 5. Rebuild BM25 from the new chunks (simpler than incremental
        //    postings update; cheap at our chunk counts). Rebuild
        //    mappings + corpus_class from the new chunks too.
        let bm25 = {
            let _g = profiler.phase("apply_diff_bm25");
            Bm25Index::build(&kept_chunks)
        };
        let (file_mapping, language_mapping) = {
            let _g = profiler.phase("apply_diff_mappings");
            build_mappings(&kept_chunks)
        };
        let corpus_class = CorpusClass::classify(&kept_chunks);

        // 6. Refresh manifest: drop deleted entries, refresh dirty
        //    entries with new (mtime, size, ino, blake3), insert new
        //    entries. blake3 requires the file bytes, so this re-reads
        //    each changed file once. Negligible (~10 µs/file warm).
        //
        //    Also apply `diff.touched_clean`: these are files whose stat
        //    tuple changed but whose content (blake3) is identical. The
        //    `diff_against_filesystem` path clones `self.manifest` before
        //    calling `diff_against_walk`, so the in-place stat-tuple
        //    refresh inside `diff_against_walk` is discarded. Without this
        //    step, every touched-but-unchanged file pays one blake3 read
        //    per reconcile cycle instead of zero. Applying the entries here
        //    — using the refreshed `FileEntry` already computed by
        //    `diff_against_walk` — restores the "one blake3 then zero"
        //    invariant on the new index.
        let mut manifest = self.manifest.clone();
        for path in &diff.deleted {
            manifest.files.remove(path);
        }
        for path in diff.new.iter().chain(diff.dirty.iter()) {
            if let Ok(entry) = FileEntry::from_path(path) {
                manifest.insert(path.clone(), entry);
            }
        }
        // Apply touched_clean refreshes: stat tuple already computed by
        // diff_against_walk; no re-read or re-hash needed.
        for (path, refreshed_entry) in &diff.touched_clean {
            if let Some(entry_mut) = manifest.files.get_mut(path) {
                entry_mut.mtime = refreshed_entry.mtime;
                entry_mut.size = refreshed_entry.size;
                entry_mut.ino = refreshed_entry.ino;
                // blake3 is unchanged (that's the definition of touched_clean)
                // but we overwrite defensively for consistency.
                entry_mut.blake3 = refreshed_entry.blake3;
            }
        }

        Ok(Self {
            chunks: kept_chunks,
            embeddings,
            bm25,
            encoder: std::sync::Arc::clone(&self.encoder),
            file_mapping,
            language_mapping,
            pagerank_lookup: self.pagerank_lookup.clone(),
            pagerank_alpha: self.pagerank_alpha,
            corpus_class,
            root: self.root.clone(),
            walk_options: self.walk_options.clone(),
            manifest,
        })
    }

    /// Compare the manifest captured at build time against the current
    /// filesystem state under [`Self::root`], using the same
    /// [`WalkOptions`] used for the original index build.
    ///
    /// Returns a [`Diff`] enumerating dirty, new, and deleted files.
    /// A zero-cost ([`Diff::is_empty`]) result means the index is
    /// up-to-date and no rebuild is needed.
    ///
    /// # Cost
    ///
    /// Walk + per-file `stat()` for the cheap-path files (typically all
    /// of them between successive queries). Blake3 verification is paid
    /// only on the rare files where the stat tuple mismatches. On a
    /// 200-file repo with no changes: sub-millisecond. On a 92k-file
    /// repo with no changes: ~100-130 ms (the walk dominates).
    ///
    /// # Mutation
    ///
    /// This method takes `&self` and works on a clone of the manifest,
    /// so the optimization of "refresh touched-but-unchanged stat
    /// tuples" from [`diff_against_walk`] is discarded here. In
    /// practice that means a file repeatedly touched without content
    /// change pays one blake3 read per reconcile rather than zero —
    /// negligible at our file sizes.
    #[must_use]
    pub fn diff_against_filesystem(&self) -> Diff {
        let files = collect_files_with_options(&self.root, &self.walk_options);
        let mut manifest = self.manifest.clone();
        diff_against_walk(&mut manifest, &files)
    }

    /// Canonical root the index was built against.
    #[must_use]
    pub fn root(&self) -> &Path {
        &self.root
    }

    /// Walk options captured at build time.
    #[must_use]
    pub fn walk_options(&self) -> &WalkOptions {
        &self.walk_options
    }

    /// Manifest of tracked files (read-only access).
    #[must_use]
    pub fn manifest(&self) -> &Manifest {
        &self.manifest
    }

    /// The index's corpus classification, computed at build time.
    ///
    /// Used by the MCP rerank gate to decide whether the L-12
    /// cross-encoder fires on a given query.
    #[must_use]
    pub fn corpus_class(&self) -> CorpusClass {
        self.corpus_class
    }

    /// Number of indexed chunks.
    #[must_use]
    pub fn len(&self) -> usize {
        self.chunks.len()
    }

    /// Whether the index has zero chunks.
    #[must_use]
    pub fn is_empty(&self) -> bool {
        self.chunks.is_empty()
    }

    /// Indexed chunks (read-only access).
    #[must_use]
    pub fn chunks(&self) -> &[CodeChunk] {
        &self.chunks
    }

    /// Indexed embeddings (read-only access).
    ///
    /// `Array2<f32>` of shape `[n_chunks, hidden_dim]`, row-major. Row
    /// `i` is the L2-normalized embedding of chunk `i`, so cosine
    /// similarity reduces to a dot product. Callers that need their
    /// own similarity arithmetic (`find_similar`, `find_duplicates`)
    /// should use `embeddings.row(i)` for a single-row view or
    /// `embeddings.dot(&query)` for a one-call BLAS GEMV.
    #[must_use]
    pub fn embeddings(&self) -> &ndarray::Array2<f32> {
        &self.embeddings
    }

    /// Search the index and return ranked `(chunk_index, score)` pairs.
    ///
    /// `mode = SearchMode::Hybrid` (default) fuses semantic + BM25 via
    /// RRF; `Semantic` and `Keyword` use one signal each.
    ///
    /// `filter_languages` and `filter_paths` build a selector mask
    /// that restricts retrieval to chunks in the named files /
    /// languages.
    #[must_use]
    pub fn search(
        &self,
        query: &str,
        top_k: usize,
        mode: SearchMode,
        alpha: Option<f32>,
        filter_languages: Option<&[String]>,
        filter_paths: Option<&[String]>,
    ) -> Vec<(usize, f32)> {
        if self.is_empty() || query.trim().is_empty() {
            return Vec::new();
        }
        let selector = self.build_selector(filter_languages, filter_paths);

        let raw = match mode {
            SearchMode::Keyword => search_bm25(query, &self.bm25, top_k, selector.as_deref()),
            SearchMode::Semantic => {
                let q_emb = self.encoder.encode_query(query);
                search_semantic(&q_emb, &self.embeddings, top_k, selector.as_deref())
            }
            SearchMode::Hybrid => {
                let q_emb = self.encoder.encode_query(query);
                search_hybrid(
                    query,
                    &q_emb,
                    &self.embeddings,
                    &self.chunks,
                    &self.bm25,
                    top_k,
                    alpha,
                    selector.as_deref(),
                )
            }
        };

        self.apply_pagerank_layer(raw)
    }

    /// Build a selector mask from optional language/path filters.
    /// Returns `None` when no filters are set (search runs over the
    /// full corpus).
    fn build_selector(
        &self,
        filter_languages: Option<&[String]>,
        filter_paths: Option<&[String]>,
    ) -> Option<Vec<usize>> {
        let mut selector: Vec<usize> = Vec::new();
        if let Some(langs) = filter_languages {
            for lang in langs {
                if let Some(ids) = self.language_mapping.get(lang) {
                    selector.extend(ids.iter().copied());
                }
            }
        }
        if let Some(paths) = filter_paths {
            for path in paths {
                if let Some(ids) = self.file_mapping.get(path) {
                    selector.extend(ids.iter().copied());
                }
            }
        }
        if selector.is_empty() {
            None
        } else {
            selector.sort_unstable();
            selector.dedup();
            Some(selector)
        }
    }

    /// Layer ripvec's PageRank boost on top of semble's ranked results.
    ///
    /// No-op when `pagerank_lookup` is `None` or the boost strength
    /// is zero. Otherwise re-uses
    /// [`crate::hybrid::boost_with_pagerank`] so the PageRank semantic
    /// stays consistent with ripvec's other code paths.
    fn apply_pagerank_layer(&self, mut results: Vec<(usize, f32)>) -> Vec<(usize, f32)> {
        let Some(lookup) = &self.pagerank_lookup else {
            return results;
        };
        if results.is_empty() || self.pagerank_alpha <= 0.0 {
            return results;
        }
        // Uses the shared `ranking::PageRankBoost` layer for behavioral
        // parity with the BERT CLI, MCP `search_code`, and LSP paths.
        // All five callers now apply the same sigmoid-on-percentile
        // curve.
        // `lookup` is `Arc<HashMap<_,_>>`; cloning the Arc is a pointer
        // bump, not a HashMap copy. The earlier `lookup.clone()` here
        // cloned the entire map per query (~10K String allocations on
        // a 1M-chunk corpus).
        let layers: Vec<Box<dyn crate::ranking::RankingLayer>> = vec![Box::new(
            crate::ranking::PageRankBoost::new(std::sync::Arc::clone(lookup), self.pagerank_alpha),
        )];
        crate::ranking::apply_chain(&mut results, &self.chunks, &layers);
        results
    }
}

impl crate::searchable::SearchableIndex for RipvecIndex {
    fn chunks(&self) -> &[CodeChunk] {
        RipvecIndex::chunks(self)
    }

    /// Trait-shape search: text-only, no engine-specific knobs.
    ///
    /// The trait surface is the LSP-callers' common ground. Filters
    /// (language, path) and the alpha auto-detect override are not
    /// surfaced through the trait because no LSP module uses them.
    fn search(&self, query_text: &str, top_k: usize, mode: SearchMode) -> Vec<(usize, f32)> {
        RipvecIndex::search(self, query_text, top_k, mode, None, None, None)
    }

    /// Use chunk `chunk_idx`'s own embedding as the query vector and
    /// rank everything else by cosine similarity (semantic-only) or
    /// blend with BM25 (hybrid). Falls back to text-only keyword
    /// search when the chunk index is out of range.
    ///
    /// Mirrors the [`HybridIndex`] equivalent so `goto_definition`
    /// and `goto_implementation` work identically across engines.
    fn search_from_chunk(
        &self,
        chunk_idx: usize,
        query_text: &str,
        top_k: usize,
        mode: SearchMode,
    ) -> Vec<(usize, f32)> {
        // RipvecIndex stores embeddings; if the source chunk is in
        // range we can rank by similarity against its vector. Out of
        // range or keyword-only mode: fall back to text search.
        if chunk_idx >= self.embeddings().nrows() {
            return RipvecIndex::search(
                self,
                query_text,
                top_k,
                SearchMode::Keyword,
                None,
                None,
                None,
            );
        }
        match mode {
            SearchMode::Keyword => RipvecIndex::search(
                self,
                query_text,
                top_k,
                SearchMode::Keyword,
                None,
                None,
                None,
            ),
            SearchMode::Semantic | SearchMode::Hybrid => {
                // Cosine via dot product over L2-normalized rows.
                // Parallel sgemv across row-shards to saturate
                // aggregate memory bandwidth instead of the single-core
                // sgemv ceiling.
                let source = self.embeddings().row(chunk_idx);
                let scores =
                    crate::encoder::ripvec::hybrid::parallel_sgemv(self.embeddings(), &source);
                let mut scored: Vec<(usize, f32)> = scores
                    .iter()
                    .enumerate()
                    .filter(|(i, _)| *i != chunk_idx)
                    .map(|(i, &s)| (i, s))
                    .collect();
                if scored.len() > top_k {
                    scored.select_nth_unstable_by(top_k - 1, |a, b| {
                        b.1.total_cmp(&a.1).then_with(|| a.0.cmp(&b.0))
                    });
                    scored.truncate(top_k);
                }
                scored.sort_unstable_by(|a, b| b.1.total_cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
                scored
            }
        }
    }

    fn as_any(&self) -> &dyn std::any::Any {
        self
    }
}

/// Build (file_path → chunk indices, language → chunk indices) mappings.
/// Build the per-file manifest by walking `root` with `walk_options`
/// and stat + read + blake3 each file. Used at index construction; on
/// reconcile, [`RipvecIndex::diff_against_filesystem`] uses the cheap
/// stat-tuple path and only re-reads files whose tuple mismatches the
/// stored entry.
///
/// Files that can't be read or stat'd are silently skipped; they will
/// re-appear in the diff as `new` if they become readable later, or
/// as missing on the next reconcile.
fn build_manifest(root: &Path, walk_options: &WalkOptions) -> Manifest {
    let mut manifest = Manifest::new();
    let files = collect_files_with_options(root, walk_options);
    for path in files {
        let (Ok(metadata), Ok(bytes)) = (std::fs::metadata(&path), std::fs::read(&path)) else {
            continue;
        };
        let entry = FileEntry::from_bytes(&metadata, &bytes);
        manifest.insert(path, entry);
    }
    manifest
}

fn build_mappings(
    chunks: &[CodeChunk],
) -> (HashMap<String, Vec<usize>>, HashMap<String, Vec<usize>>) {
    let mut file_to_id: HashMap<String, Vec<usize>> = HashMap::new();
    let mut lang_to_id: HashMap<String, Vec<usize>> = HashMap::new();
    for (i, chunk) in chunks.iter().enumerate() {
        file_to_id
            .entry(chunk.file_path.clone())
            .or_default()
            .push(i);
        // The semble port's chunker stores language inferentially (via
        // extension); the per-chunk `language` field isn't populated on
        // this path. The mapping is keyed on file extension as a proxy
        // so `filter_languages: Some(&["rs"])` works.
        if let Some(ext) = Path::new(&chunk.file_path)
            .extension()
            .and_then(|e| e.to_str())
        {
            lang_to_id.entry(ext.to_string()).or_default().push(i);
        }
    }
    (file_to_id, lang_to_id)
}

#[cfg(test)]
mod tests {
    use super::*;

    /// Test-only constructor that bypasses `from_root` to allow unit
    /// tests to inject pre-built state (chunks, embeddings, mappings,
    /// manifest) without requiring a real model download.
    ///
    /// For tests that call `apply_diff` with a non-empty `diff.new` or
    /// `diff.dirty`, the caller must supply a real encoder because
    /// `apply_diff` calls `encoder.embed_paths`.
    #[allow(clippy::too_many_arguments)]
    fn new_for_test(
        chunks: Vec<crate::chunk::CodeChunk>,
        embeddings: ndarray::Array2<f32>,
        encoder: std::sync::Arc<StaticEncoder>,
        file_mapping: HashMap<String, Vec<usize>>,
        language_mapping: HashMap<String, Vec<usize>>,
        manifest: Manifest,
        root: std::path::PathBuf,
        walk_options: WalkOptions,
    ) -> RipvecIndex {
        let bm25 = Bm25Index::build(&chunks);
        let corpus_class = CorpusClass::classify(&chunks);
        RipvecIndex {
            chunks,
            embeddings,
            bm25,
            encoder,
            file_mapping,
            language_mapping,
            pagerank_lookup: None,
            pagerank_alpha: 0.0,
            corpus_class,
            root,
            walk_options,
            manifest,
        }
    }

    /// Compile-time check that `RipvecIndex` carries the right method
    /// shape for the CLI to call.
    #[test]
    fn semble_index_search_signature_compiles() {
        fn shape_check(
            idx: &RipvecIndex,
            query: &str,
            top_k: usize,
            mode: SearchMode,
        ) -> Vec<(usize, f32)> {
            idx.search(query, top_k, mode, None, None, None)
        }
        // Reference to keep type-check live across dead-code analysis.
        let _ = shape_check;
    }

    /// `behavior:pagerank-no-op-when-graph-absent` — when constructed
    /// without a PageRank lookup, the layer is a pure pass-through.
    /// (Asserted via the `apply_pagerank_layer` early-return path.)
    #[test]
    fn pagerank_layer_no_op_when_graph_absent() {
        // We can't easily build a RipvecIndex without a real encoder
        // (which requires a model download). Instead, exercise the
        // pass-through logic on a hand-built struct via the private
        // method. The function returns its input unchanged when
        // pagerank_lookup is None.
        //
        // Structural assertion: apply_pagerank_layer's first match
        // statement returns the input directly when lookup is None;
        // this is a single-branch invariant verified by inspection.
        // Behavioural verification is part of P5.1's parity test.
        let _ = "see apply_pagerank_layer docs";
    }

    /// Corner case: a file appears in `diff.new` (absent from manifest)
    /// but `file_mapping` still holds stale chunk indices for it from a
    /// prior partial reconcile. Without the R4.1 fix, `apply_diff` skips
    /// clearing those stale chunks before re-embedding → duplicates.
    ///
    /// Gated `#[ignore]` because `apply_diff` calls `encoder.embed_paths`
    /// for files in `diff.new`, which requires the Model2Vec weights.
    /// Run once model is cached:
    ///   `cargo test -p ripvec-core apply_diff_idempotent -- --ignored`
    #[test]
    #[ignore = "requires Model2Vec download (~32 MB on first run)"]
    fn apply_diff_idempotent_when_new_file_already_has_chunks() {
        use crate::encoder::ripvec::dense::{DEFAULT_MODEL_REPO, StaticEncoder};
        use crate::profile::Profiler;
        use std::fs;

        let encoder = StaticEncoder::from_pretrained(DEFAULT_MODEL_REPO).expect("encoder load");
        let encoder_arc = std::sync::Arc::new(encoder);

        // Temporary corpus: one file (file_a.rs).
        let tmp = tempfile::TempDir::new().unwrap();
        let file_a = tmp.path().join("file_a.rs");
        fs::write(
            &file_a,
            "pub fn alpha() -> u32 { 1 }\npub fn beta() -> u32 { 2 }\n",
        )
        .unwrap();

        // Embed file_a.rs once to obtain its canonical chunks/embeddings.
        let (real_chunks, real_embs) = encoder_arc
            .embed_paths(tmp.path(), std::slice::from_ref(&file_a), &Profiler::noop())
            .expect("embed_paths");
        let n_real = real_chunks.len();
        assert!(n_real > 0, "file_a.rs must produce at least one chunk");

        let hidden_dim = real_embs[0].len();
        let mut flat: Vec<f32> = Vec::with_capacity(n_real * hidden_dim);
        for row in &real_embs {
            flat.extend(row);
        }
        let embeddings = ndarray::Array2::from_shape_vec((n_real, hidden_dim), flat).unwrap();

        // file_mapping holds stale indices pointing at file_a.rs chunks.
        let rel_key = "file_a.rs".to_string();
        let indices: Vec<usize> = (0..n_real).collect();
        let file_mapping = HashMap::from([(rel_key, indices)]);

        // Manifest is EMPTY: simulates a prior reconcile whose manifest
        // update failed, so diff_against_filesystem classifies file_a.rs
        // as "new" even though file_mapping still references its chunks.
        let manifest = Manifest::new();

        let index = new_for_test(
            real_chunks,
            embeddings,
            std::sync::Arc::clone(&encoder_arc),
            file_mapping,
            HashMap::new(),
            manifest,
            tmp.path().to_path_buf(),
            WalkOptions::default(),
        );

        let diff = index.diff_against_filesystem();
        assert!(
            diff.new.iter().any(|p| p.ends_with("file_a.rs")),
            "file_a.rs must appear in diff.new when manifest is empty; got {:?}",
            diff.new
        );
        assert!(diff.dirty.is_empty(), "no dirty expected");
        assert!(diff.deleted.is_empty(), "no deleted expected");

        // With the fix (diff.new also processed in removed_indices), stale
        // chunks are dropped before re-embedding → chunk count equals
        // one fresh-embed pass. Without the fix, old + new chunks both
        // survive → count is doubled.
        let updated = index
            .apply_diff(&diff, &Profiler::noop())
            .expect("apply_diff");

        let file_a_count = updated
            .chunks()
            .iter()
            .filter(|c| c.file_path.ends_with("file_a.rs"))
            .count();

        assert_eq!(
            file_a_count, n_real,
            "file_a.rs chunk count must equal one fresh-embed pass ({n_real}); \
             got {file_a_count} — stale chunks from file_mapping not cleared"
        );
        assert_eq!(
            updated.embeddings().nrows(),
            updated.chunks().len(),
            "embeddings row count must match chunk count"
        );
    }

    /// Derived: applying an empty diff twice must yield identical chunk
    /// counts — no accumulation from repeated no-op reconciles.
    ///
    /// Gated `#[ignore]` because building a real index requires the
    /// Model2Vec encoder (~32 MB).
    #[test]
    #[ignore = "requires Model2Vec download (~32 MB on first run)"]
    fn apply_diff_no_duplicate_chunks_after_two_passes() {
        use crate::embed::SearchConfig;
        use crate::encoder::ripvec::dense::{DEFAULT_MODEL_REPO, StaticEncoder};
        use crate::profile::Profiler;
        use std::fs;

        let tmp = tempfile::TempDir::new().unwrap();
        fs::write(
            tmp.path().join("main.rs"),
            "fn main() { println!(\"hello\"); }\n",
        )
        .unwrap();

        let encoder = StaticEncoder::from_pretrained(DEFAULT_MODEL_REPO).expect("encoder load");
        let cfg = SearchConfig {
            batch_size: 32,
            max_tokens: 512,
            chunk: crate::chunk::ChunkConfig {
                max_chunk_bytes: 4096,
                window_size: 2048,
                window_overlap: 512,
            },
            text_mode: false,
            cascade_dim: None,
            file_type: None,
            exclude_extensions: Vec::new(),
            include_extensions: Vec::new(),
            ignore_patterns: Vec::new(),
            scope: crate::embed::Scope::All,
            mode: crate::hybrid::SearchMode::Hybrid,
        };
        let index = RipvecIndex::from_root(tmp.path(), encoder, &cfg, &Profiler::noop(), None, 0.0)
            .expect("from_root");

        let original_count = index.chunks().len();

        let diff1 = index.diff_against_filesystem();
        assert!(diff1.is_empty(), "fresh index must yield empty diff");
        let pass1 = index
            .apply_diff(&diff1, &Profiler::noop())
            .expect("apply_diff pass 1");
        assert_eq!(
            pass1.chunks().len(),
            original_count,
            "chunk count must be unchanged after empty-diff pass 1"
        );

        let diff2 = pass1.diff_against_filesystem();
        assert!(
            diff2.is_empty(),
            "pass1 against unchanged FS must yield empty diff"
        );
        let pass2 = pass1
            .apply_diff(&diff2, &Profiler::noop())
            .expect("apply_diff pass 2");
        assert_eq!(
            pass2.chunks().len(),
            original_count,
            "chunk count must be unchanged after empty-diff pass 2"
        );
    }
}