lean_ctx/core/bm25_index/

mod.rs

use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::time::UNIX_EPOCH;

use serde::{Deserialize, Serialize};
mod build;
mod carry_over;
mod chunking;
pub use chunking::*;
mod coordinator;
pub mod coverage;
mod evict;
pub(crate) mod external_origin;
pub use coordinator::{SearchIndexBuildProgress, get_or_start_build};
mod provider_origin;
pub use provider_origin::ChunkOrigins;
mod served_ranking;
pub mod source_gate;
#[cfg(test)]
mod tests;
mod walk;
pub use walk::is_code_file;
use walk::{admitted_scoped_source_paths, list_code_files};
#[cfg(test)]
use walk::{list_code_files_with, list_scoped_code_files};

pub(crate) const MAX_BM25_FILES: usize = 5000;
const CHUNK_COUNT_WARNING: usize = 50_000;
const ZSTD_LEVEL: i32 = 9;

const DEFAULT_BM25_IGNORES: &[&str] = &[
    "vendor/**",
    "dist/**",
    "build/**",
    "public/vendor/**",
    "public/js/**",
    "public/css/**",
    "public/build/**",
    ".next/**",
    ".nuxt/**",
    "__pycache__/**",
    "*.min.js",
    "*.min.css",
    "*.bundle.js",
    "*.chunk.js",
];

fn max_bm25_cache_bytes() -> u64 {
    // Single source of truth: `Config::bm25_max_cache_mb_effective` (env override
    // › explicit config › disk-budget › generous default). Decoupled from the RAM
    // profile so large repos persist instead of rebuilding forever (issue #249).
    let mb = std::env::var("LEAN_CTX_BM25_MAX_CACHE_MB")
        .ok()
        .and_then(|v| v.parse::<u64>().ok())
        .unwrap_or_else(|| crate::core::config::Config::load().bm25_max_cache_mb_effective());
    mb * 1024 * 1024
}

/// Effective on-disk ceiling (bytes) for the persisted BM25 index. Single source
/// of truth shared with `doctor` so its "oversized index" warning matches what
/// `save`/`load` actually enforce.
pub fn persist_ceiling_bytes() -> u64 {
    max_bm25_cache_bytes()
}

/// Outcome of persisting a BM25 index to disk. Distinguishes a real write from a
/// size-capped refusal so callers never mistake "refused to persist" for
/// success (the bug behind the perpetual "index warming" report, issue #249).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SaveOutcome {
    /// Written to disk. Carries the compressed (zstd) size in bytes.
    Persisted { compressed_bytes: u64 },
    /// Built fine but NOT written — the compressed size exceeds the disk
    /// ceiling. The in-memory index is still usable for this process; callers
    /// should surface the remedy (raise the cap / add ignore patterns) instead
    /// of silently rebuilding on every call.
    SkippedTooLarge {
        compressed_bytes: u64,
        limit_bytes: u64,
    },
}

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct CodeChunk {
    pub file_path: String,
    pub symbol_name: String,
    pub kind: ChunkKind,
    pub start_line: usize,
    pub end_line: usize,
    pub content: String,
    #[serde(default)]
    pub tokens: Vec<String>,
    pub token_count: usize,
}

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub enum ChunkKind {
    Function,
    Struct,
    Impl,
    Module,
    Class,
    Method,
    Other,
    // -- External source kinds (Context Engine) --
    Issue,
    PullRequest,
    WikiPage,
    DbSchema,
    ApiEndpoint,
    Ticket,
    ExternalOther,
}

#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
pub struct IndexedFileState {
    pub mtime_ms: u64,
    pub size_bytes: u64,
}

impl IndexedFileState {
    fn from_path(path: &Path) -> Option<Self> {
        let meta = path.metadata().ok()?;
        let size_bytes = meta.len();
        let mtime_ms = meta
            .modified()
            .ok()
            .and_then(|t| t.duration_since(UNIX_EPOCH).ok())
            .map(|d| d.as_millis() as u64)?;
        Some(Self {
            mtime_ms,
            size_bytes,
        })
    }
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BM25Index {
    pub chunks: Vec<CodeChunk>,
    pub inverted: HashMap<String, Vec<(usize, f64)>>,
    pub avg_doc_len: f64,
    pub doc_count: usize,
    pub doc_freqs: HashMap<String, usize>,
    #[serde(default)]
    pub files: HashMap<String, IndexedFileState>,
    /// Content fingerprint of every directory that contained an indexed file
    /// when this index was built, keyed by root-relative path (`""` = the root
    /// itself). Sampling tracked *files* can never reveal a file that was never
    /// indexed, so the fast staleness check would keep serving a partial index
    /// forever after new files appear (#1724). Creating or removing an entry
    /// bumps its parent directory's mtime on every filesystem we support, so
    /// comparing this snapshot detects added/deleted files in one stat per
    /// directory instead of a full ingest walk.
    #[serde(default)]
    pub dirs: HashMap<String, u64>,
    /// Digest of the Context Gateway policy the chunk contents were admitted
    /// under. An index built under another policy (or before admission
    /// existed — its postcard layout lacks this field and no longer loads) is
    /// rebuilt in full, never reused (G5, E3).
    #[serde(default)]
    pub admission_policy: Option<String>,
    /// True once `shrink_resident_content_to_snippet` has trimmed each chunk's
    /// `content` down to the snippet lines. Resident-only RAM-saving state: never
    /// persisted (`skip`) so the on-disk index keeps full content, and a reload
    /// always starts as `false`. Guards the embedding pass against re-embedding
    /// truncated bodies (see `ensure_embeddings`).
    #[serde(default, skip)]
    pub content_truncated: bool,
    /// Project root of an index its store loaded or handed out (`load`,
    /// `load_or_build*`): `search` then serves a chunk only while the admitted
    /// reader still returns its text (`source_gate`, G04). Resident only;
    /// `None` for a view built in the running call.
    #[serde(default, skip)]
    pub source_root: Option<PathBuf>,
    /// Where each chunk without a readable text file came from, by chunk
    /// `file_path`. A reference, never a grant (`provider_origin`). Kept
    /// beside the index file, not in it: the postcard layout is unchanged.
    #[serde(default, skip)]
    pub provider_origins: std::sync::Arc<ChunkOrigins>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SearchResult {
    pub chunk_idx: usize,
    pub score: f64,
    pub file_path: String,
    pub symbol_name: String,
    pub kind: ChunkKind,
    pub start_line: usize,
    pub end_line: usize,
    pub snippet: String,
}

const BM25_K1: f64 = 1.2;
const BM25_B: f64 = 0.75;

fn structural_query_boost(query: &str, chunk: &CodeChunk) -> f64 {
    let raw_terms: std::collections::HashSet<String> = query
        .split(|c: char| !(c.is_alphanumeric() || c == '_'))
        .filter(|term| term.len() >= 3)
        .map(str::to_ascii_lowercase)
        .collect();
    let path_terms: std::collections::HashSet<String> = tokenize(&chunk.file_path)
        .into_iter()
        .map(|term| term.to_ascii_lowercase())
        .collect();

    let mut boost = 0.0;
    if raw_terms.contains(&chunk.symbol_name.to_ascii_lowercase()) {
        boost += 32.0;
    }
    boost += raw_terms.intersection(&path_terms).count() as f64 * 8.0;

    let has_identifier = query.split_whitespace().any(|term| {
        let has_lower = term.chars().any(char::is_lowercase);
        let has_upper = term.chars().any(char::is_uppercase);
        (has_lower && has_upper) || term.contains('_') || term.contains('.')
    });
    if has_identifier {
        let extension = std::path::Path::new(&chunk.file_path)
            .extension()
            .and_then(std::ffi::OsStr::to_str)
            .unwrap_or_default()
            .to_ascii_lowercase();
        if matches!(extension.as_str(), "md" | "mdx" | "rst" | "txt" | "adoc") {
            boost -= 8.0;
        } else if !extension.is_empty() {
            boost += 4.0;
        }
    }
    boost
}

impl Default for BM25Index {
    fn default() -> Self {
        Self::new()
    }
}

/// Sources admitted, then chunked in parallel, per batch of a fresh view.
const ADMITTED_BATCH: usize = 256;
/// Chunks one file may add to a fresh view.
const FILE_CHUNK_COUNT: usize = 4096;

type ChunkedFile = (Vec<CodeChunk>, usize, usize);

/// The chunks of one admitted file in line order, charged against the given
/// budgets, as a fresh view adds them.
fn extract_sorted_chunks(
    path: &str,
    content: &str,
    chunk_bytes: &mut usize,
    chunk_count: &mut usize,
) -> Result<Vec<CodeChunk>, String> {
    let mut chunks = extract_admitted_chunks(path, content, chunk_bytes, chunk_count)?;
    chunks.sort_by(|a, b| {
        (a.start_line, a.end_line, &a.symbol_name).cmp(&(b.start_line, b.end_line, &b.symbol_name))
    });
    Ok(chunks)
}

/// Each admitted file's own byte allowance and its chunks under that
/// allowance and the per-file chunk count, computed in parallel: chunking is
/// a pure function of path and text. `None` for a file the memory guard
/// stopped before it was chunked; the fold then chunks it itself.
fn parallel_admitted_chunks(
    admitted: &[(&str, String)],
) -> Vec<(usize, Option<Result<ChunkedFile, String>>)> {
    use rayon::prelude::*;
    admitted
        .par_iter()
        .map(|(path, content)| {
            let full_allowance =
                (64 * 1024 * 1024usize).min(content.len().saturating_mul(5).max(64 * 1024));
            if crate::core::memory_guard::abort_requested() {
                return (full_allowance, None);
            }
            let mut bytes = full_allowance;
            let mut count = FILE_CHUNK_COUNT;
            let result = extract_sorted_chunks(path, content, &mut bytes, &mut count)
                .map(|chunks| (chunks, full_allowance - bytes, FILE_CHUNK_COUNT - count));
            (full_allowance, Some(result))
        })
        .collect()
}

/// One load–change–save of a persisted index at a time in this process. Two
/// writers (two provider ingests a moment apart, an eviction after a search)
/// otherwise each saved their own copy through the one temp file and could
/// leave an index no reader loads, or drop the other's chunks.
static PERSIST: std::sync::Mutex<()> = std::sync::Mutex::new(());

pub(crate) fn persist_lock() -> std::sync::MutexGuard<'static, ()> {
    PERSIST
        .lock()
        .unwrap_or_else(std::sync::PoisonError::into_inner)
}

impl BM25Index {
    pub fn new() -> Self {
        Self {
            chunks: Vec::new(),
            inverted: HashMap::new(),
            avg_doc_len: 0.0,
            doc_count: 0,
            doc_freqs: HashMap::new(),
            files: HashMap::new(),
            dirs: HashMap::new(),
            admission_policy: None,
            content_truncated: false,
            source_root: None,
            provider_origins: std::sync::Arc::default(),
        }
    }

    /// Approximate heap memory used by this index in bytes.
    pub fn memory_usage_bytes(&self) -> usize {
        let chunks_size: usize = self
            .chunks
            .iter()
            .map(|c| {
                c.content.len()
                    + c.file_path.len()
                    + c.symbol_name.len()
                    + c.tokens.iter().map(String::len).sum::<usize>()
                    + 64
            })
            .sum();
        let inverted_size: usize = self
            .inverted
            .iter()
            .map(|(k, v)| k.len() + v.len() * 16 + 32)
            .sum();
        let files_size: usize = self.files.keys().map(|k| k.len() + 24).sum();
        let dirs_size: usize = self.dirs.keys().map(|k| k.len() + 16).sum();
        let freqs_size: usize = self.doc_freqs.keys().map(|k| k.len() + 16).sum();
        chunks_size + inverted_size + files_size + dirs_size + freqs_size
    }

    /// Drops all in-memory data, effectively freeing heap. Index can be re-loaded from disk.
    pub fn unload(&mut self) {
        let usage = self.memory_usage_bytes();
        self.chunks = Vec::new();
        self.inverted = HashMap::new();
        self.doc_freqs = HashMap::new();
        self.files = HashMap::new();
        self.dirs = HashMap::new();
        self.provider_origins = std::sync::Arc::default();
        self.avg_doc_len = 0.0;
        self.doc_count = 0;
        tracing::info!(
            "[bm25] unloaded index, freed ~{:.1}MB",
            usage as f64 / 1_048_576.0
        );
    }

    /// Shrinks each resident chunk's `content` to its first `keep_lines` lines,
    /// reclaiming the RAM held by the full source bodies once the embedding pass
    /// has already consumed them. Query previews can only select lines still
    /// resident, so this trades preview coverage for memory in cached indexes.
    /// Fresh admitted retrieval views retain their separately budgeted content.
    ///
    /// Further shrinks resident content (chunks are already truncated to 10 lines
    /// during `add_chunk`; this method can tighten that further). Sets
    /// `content_truncated` so a later `ensure_embeddings` against this same
    /// resident index skips re-embedding (which would otherwise feed truncated
    /// bodies to the embedder).
    ///
    /// Idempotent and only ever shrinks: chunks shorter than `keep_lines` are
    /// left untouched.
    pub fn shrink_resident_content_to_snippet(&mut self, keep_lines: usize) {
        let before = self.memory_usage_bytes();
        for chunk in &mut self.chunks {
            // Cheap line-count gate: only allocate a new string when the body is
            // actually longer than the snippet window.
            if chunk.content.lines().nth(keep_lines).is_some() {
                let trimmed: String = chunk
                    .content
                    .lines()
                    .take(keep_lines)
                    .collect::<Vec<_>>()
                    .join("\n");
                chunk.content = trimmed;
                // Reclaim the spare capacity left by the larger original body.
                chunk.content.shrink_to_fit();
            }
        }
        self.content_truncated = true;
        let after = self.memory_usage_bytes();
        tracing::debug!(
            "[bm25] shrank resident content to {keep_lines} lines/chunk, freed ~{:.1}MB",
            before.saturating_sub(after) as f64 / 1_048_576.0
        );
    }

    /// Builds an index from explicit chunks (unit tests; avoids filesystem walking).
    #[cfg(any(test, feature = "test-support"))]
    pub fn from_chunks_for_test(chunks: Vec<CodeChunk>) -> Self {
        let mut index = Self::new();
        for mut chunk in chunks {
            if chunk.token_count == 0 {
                chunk.token_count = tokenize(&chunk.content).len();
            }
            index.add_chunk(chunk);
        }
        index.finalize();
        index
    }

    pub fn build_from_directory(root: &Path) -> Self {
        Self::build_from_directory_inner(root, &HashMap::new())
    }

    /// Ephemeral retrieval view: admit each complete original source before
    /// tokenization. Never consult metadata-only caches or persist this view.
    ///
    /// `subdir` / `in_scope` narrow the walk itself (see
    /// `walk::list_scoped_code_files`), so the file budget counts only sources the
    /// caller asked about.
    pub fn build_from_admitted_sources(
        root: &Path,
        subdir: Option<&str>,
        in_scope: &dyn Fn(&str) -> bool,
        admit: impl FnMut(&str) -> Result<Option<String>, String>,
    ) -> Result<Self, String> {
        let rules = CorpusRules::load();
        Self::build_scoped_from_admitted_sources(root, &rules, subdir, in_scope, admit)
    }

    fn build_scoped_from_admitted_sources(
        root: &Path,
        rules: &CorpusRules,
        subdir: Option<&str>,
        in_scope: &dyn Fn(&str) -> bool,
        mut admit: impl FnMut(&str) -> Result<Option<String>, String>,
    ) -> Result<Self, String> {
        if !is_safe_bm25_root(root) {
            return Err("source root is not safe to scan".into());
        }
        let mut index = Self::new();
        let mut chunk_bytes = 64 * 1024 * 1024;
        let mut chunk_count = 100_000;
        // Memory a finished search dropped is not pressure: return it first.
        if crate::core::memory_guard::is_under_pressure() {
            crate::core::memory_guard::reclaim_now();
        }
        let paths = admitted_scoped_source_paths(root, rules, subdir, in_scope)?;
        // Admission runs here, on the request's thread and in its scopes, one
        // source after the other. Only the chunking of what was admitted — a
        // pure function of path and text — runs in parallel, per batch; the
        // fold below applies the budgets in path order exactly as a
        // sequential build would, so the view is the same either way.
        for batch in paths.chunks(ADMITTED_BATCH) {
            let mut admitted = Vec::with_capacity(batch.len());
            for path in batch {
                // The working set of one search is freed when it ends: it is
                // refused at the hard level, not at the soft target the
                // guardian keeps idle memory under (the build reclaimed above).
                if crate::core::memory_guard::abort_requested() {
                    return Err("source search withheld under memory pressure".into());
                }
                let Some(content) = admit(path)? else {
                    continue;
                };
                if looks_minified(&content) {
                    continue;
                }
                admitted.push((path.as_str(), content));
            }
            let chunked = parallel_admitted_chunks(&admitted);
            for ((path, content), (full_allowance, unbounded)) in admitted.iter().zip(chunked) {
                let allowance = chunk_bytes.min(full_allowance);
                if !crate::core::index_admission::admit(
                    crate::core::index_admission::BuildKind::Bm25,
                    allowance as u64,
                )
                .parallel_ok
                {
                    return Err("source search exceeds available memory headroom".into());
                }
                let count_allowance = chunk_count.min(FILE_CHUNK_COUNT);
                // The parallel result holds only where this file's budgets
                // were not capped by what earlier files used; otherwise it is
                // recomputed here under the capped budgets.
                let (chunks, used_bytes, used_count) = match unbounded {
                    Some(Ok(result))
                        if allowance == full_allowance && count_allowance == FILE_CHUNK_COUNT =>
                    {
                        result
                    }
                    Some(Err(error))
                        if allowance == full_allowance && count_allowance == FILE_CHUNK_COUNT =>
                    {
                        return Err(error);
                    }
                    _ => {
                        let mut file_chunk_bytes = allowance;
                        let mut file_chunk_count = count_allowance;
                        let chunks = extract_sorted_chunks(
                            path,
                            content,
                            &mut file_chunk_bytes,
                            &mut file_chunk_count,
                        )?;
                        (
                            chunks,
                            allowance - file_chunk_bytes,
                            count_allowance - file_chunk_count,
                        )
                    }
                };
                chunk_bytes -= used_bytes;
                chunk_count -= used_count;
                for chunk in chunks {
                    if crate::core::memory_guard::abort_requested() {
                        return Err("source search withheld under memory pressure".into());
                    }
                    // This ephemeral view already charges the full admitted chunk
                    // against its byte budget; retain it for query-centred previews.
                    index.add_chunk_with_content_limit(chunk, None);
                }
            }
        }
        index.finalize();
        Ok(index)
    }

    /// Like `build_from_directory` but reuses file content from a prior scan
    /// (e.g. the graph index walk) to avoid redundant disk reads.
    pub fn build_with_content_hint(root: &Path, content_hint: &HashMap<String, String>) -> Self {
        Self::build_from_directory_inner(root, content_hint)
    }

    fn build_from_directory_inner(root: &Path, content_hint: &HashMap<String, String>) -> Self {
        let root_str = root.to_string_lossy();
        if !super::graph_index::is_safe_scan_root_public(&root_str) {
            tracing::warn!("[bm25: scan aborted for unsafe root {root_str}]");
            return Self::new();
        }
        let files = list_code_files(root);
        // Snapshot directory mtimes *before* the (slow) chunking pass: a file
        // that lands while we build then leaves its directory newer than the
        // snapshot, so the next staleness check sees it instead of silently
        // pinning the partial tree (#1724).
        let dirs = dir_states(root, &files);
        // One policy for the whole build, recorded with the index (G5).
        let admission = crate::core::context_admission::stores::StoreAdmission::current();

        // #933: parallel fast path for the common case. The per-file parse +
        // tokenize work is pure and thread-safe, so we fan it across a rayon pool
        // and merge sequentially in file order — measured 2.85 s → 0.69 s (~4x) on
        // the lean-ctx repo. Under memory pressure (or for tiny corpora, where pool
        // setup is not worth it) we fall back to the sequential build, which carries
        // the incremental early-break. #685 adds admission control: a corpus whose
        // estimated peak would blow past the guardian's hard threshold degrades to
        // the sequential build up front instead of OOMing mid-fan-out. Both paths
        // produce an identical index (see `build` module + determinism tests).
        if files.len() >= build::PARALLEL_MIN_FILES
            && !crate::core::memory_guard::is_under_pressure()
            && !crate::core::memory_guard::abort_requested()
            && crate::core::index_admission::admit_files(
                crate::core::index_admission::BuildKind::Bm25,
                root,
                &files,
            )
            .parallel_ok
        {
            let mut index = Self::build_parallel(root, content_hint, &files, &admission);
            index.dirs = dirs;
            index.admission_policy = Some(admission.digest().to_owned());
            return index;
        }
        let mut index = Self::build_sequential(root, content_hint, &files, &admission);
        index.dirs = dirs;
        index.admission_policy = Some(admission.digest().to_owned());
        index
    }

    /// Group a previous index's chunks by file, each file's list sorted by
    /// (start_line, end_line, symbol_name) — the deterministic reuse order shared
    /// by both incremental rebuild paths (and the equivalence test, so it feeds
    /// them identical inputs).
    fn group_prev_chunks_by_file(prev: &BM25Index) -> HashMap<String, Vec<CodeChunk>> {
        let mut old_by_file: HashMap<String, Vec<CodeChunk>> = HashMap::new();
        for c in &prev.chunks {
            old_by_file
                .entry(c.file_path.clone())
                .or_default()
                .push(c.clone());
        }
        for v in old_by_file.values_mut() {
            v.sort_by(|a, b| {
                a.start_line
                    .cmp(&b.start_line)
                    .then_with(|| a.end_line.cmp(&b.end_line))
                    .then_with(|| a.symbol_name.cmp(&b.symbol_name))
            });
        }
        old_by_file
    }

    pub fn rebuild_incremental(root: &Path, prev: &BM25Index) -> Self {
        // Reused chunks must have been admitted under the current policy.
        let admission = crate::core::context_admission::stores::StoreAdmission::current();
        if prev.admission_policy.as_deref() != Some(admission.digest()) {
            return Self::build_from_directory(root);
        }
        let old_by_file = Self::group_prev_chunks_by_file(prev);
        let files = list_code_files(root);
        let dirs = dir_states(root, &files);

        // #581: mirror `build()`'s dispatch. The edit loop is the hottest path in
        // daily use, and its serial cost is dominated by re-tokenizing the *many
        // unchanged* files' reused chunks — not the few changed ones. So the
        // parallel path fans the entire tokenization (changed files via
        // `prepare_file`, unchanged via re-`prepare_chunk`) across the rayon pool
        // and merges sequentially in file order. The sequential path keeps the
        // per-file memory-pressure early-break and serves small corpora / pressure.
        // #685: oversized corpora degrade to sequential via admission control.
        // Both produce an identical index (see determinism tests).
        if files.len() >= build::PARALLEL_MIN_FILES
            && !crate::core::memory_guard::is_under_pressure()
            && !crate::core::memory_guard::abort_requested()
            && crate::core::index_admission::admit_files(
                crate::core::index_admission::BuildKind::Bm25,
                root,
                &files,
            )
            .parallel_ok
        {
            let mut index =
                Self::rebuild_incremental_parallel(root, prev, &old_by_file, &files, &admission);
            index.dirs = dirs;
            index.admission_policy = Some(admission.digest().to_owned());
            return index;
        }
        let mut index =
            Self::rebuild_incremental_sequential(root, prev, &old_by_file, &files, &admission);
        index.dirs = dirs;
        index.admission_policy = Some(admission.digest().to_owned());
        index
    }

    /// Sequential incremental rebuild with per-file memory-pressure guards. Reuses
    /// unchanged files' chunks and re-extracts changed ones. Used for small
    /// corpora and as the safe fallback under memory pressure.
    pub(crate) fn rebuild_incremental_sequential(
        root: &Path,
        prev: &BM25Index,
        old_by_file: &HashMap<String, Vec<CodeChunk>>,
        files: &[String],
        admission: &crate::core::context_admission::stores::StoreAdmission,
    ) -> Self {
        let mut index = Self::new();
        const MAX_FILE_SIZE_BYTES: u64 = 2 * 1024 * 1024;
        let root_key = root.to_string_lossy().to_string();
        let total = files.len() as u64;
        crate::core::index_progress::report_bm25(&root_key, 0, total);

        for (i, rel) in files.iter().enumerate() {
            if i.is_multiple_of(16) || i + 1 == files.len() {
                crate::core::index_progress::report_bm25(&root_key, (i + 1) as u64, total);
            }
            if i.is_multiple_of(500) && crate::core::memory_guard::is_under_pressure() {
                tracing::warn!(
                    "[bm25: stopping incremental rebuild at file {i}/{} due to memory pressure]",
                    files.len()
                );
                break;
            }

            let abs = root.join(rel);
            let Some(state) = IndexedFileState::from_path(&abs) else {
                continue;
            };

            let unchanged = prev.files.get(rel).is_some_and(|old| *old == state);
            if unchanged
                && let Some(chunks) = old_by_file.get(rel)
                && chunks.first().is_some_and(|c| !c.content.is_empty())
            {
                for chunk in chunks {
                    index.add_chunk(chunk.clone());
                }
                index.files.insert(rel.clone(), state);
                continue;
            }

            if state.size_bytes > MAX_FILE_SIZE_BYTES {
                continue;
            }
            let content = if crate::core::extractors::is_binary_document(&abs) {
                match std::fs::read(&abs) {
                    Ok(bytes) => crate::core::extractors::extract(&abs, &bytes).text,
                    Err(_) => continue,
                }
            } else {
                // An incompletely decoded source (replaced or dropped bytes)
                // is never a complete source: it does not enter the index.
                match crate::core::text_decode::read_text_checked(&abs) {
                    Ok(decoded) if decoded.complete() => decoded.text,
                    _ => continue,
                }
            };
            if content.is_empty() {
                continue;
            }
            // Derived store: only admitted text, never restricted (E3).
            let Some(content) = admission.admit(&content, &abs) else {
                continue;
            };
            // #1739: see `build::prepare_file` — same predicate, same position.
            if looks_minified(&content) {
                tracing::debug!("[bm25: skipping minified payload {rel}]");
                continue;
            }
            let mut chunks = extract_chunks(rel, &content);
            chunks.sort_by(|a, b| {
                a.start_line
                    .cmp(&b.start_line)
                    .then_with(|| a.end_line.cmp(&b.end_line))
                    .then_with(|| a.symbol_name.cmp(&b.symbol_name))
            });
            for chunk in chunks {
                index.add_chunk(chunk);
            }
            index.files.insert(rel.clone(), state);
        }

        index.finalize();
        index
    }

    fn add_chunk(&mut self, chunk: CodeChunk) {
        self.add_chunk_with_content_limit(chunk, Some(10));
    }

    fn add_chunk_with_content_limit(&mut self, mut chunk: CodeChunk, keep_lines: Option<usize>) {
        let idx = self.chunks.len();

        let enriched = enrich_for_bm25(&chunk);
        let tokens = chunking::index_terms(&enriched);
        for token in &tokens {
            let lower = token.to_lowercase();
            let postings = self.inverted.entry(lower.clone()).or_default();
            if postings.last().map(|(last_idx, _)| *last_idx) != Some(idx) {
                *self.doc_freqs.entry(lower).or_insert(0) += 1;
            }
            postings.push((idx, 1.0));
        }

        // #790: truncate content to snippet AFTER tokenization — full text was
        // used for BM25 scoring above; stored content is only for result display.
        if let Some(keep_lines) = keep_lines.filter(|&n| chunk.content.lines().nth(n).is_some()) {
            chunk.content = chunk
                .content
                .lines()
                .take(keep_lines)
                .collect::<Vec<_>>()
                .join("\n");
            chunk.content.shrink_to_fit();
        }

        self.chunks.push(CodeChunk {
            token_count: tokens.len(),
            tokens: Vec::new(),
            ..chunk
        });
    }

    fn finalize(&mut self) {
        self.doc_count = self.chunks.len();
        if self.doc_count == 0 {
            return;
        }

        let total_len: usize = self.chunks.iter().map(|c| c.token_count).sum();
        self.avg_doc_len = total_len as f64 / self.doc_count as f64;
    }

    /// Every chunk matching `query`, best first. With `excluded` (one flag
    /// per chunk) the flagged chunks do not enter the ranking at all: they
    /// get no score and are left out of the corpus statistics (document
    /// count, average length, document frequencies), so the result is the
    /// ranking of an index that never held them (`served_ranking`).
    fn rank(&self, query: &str, excluded: Option<&[bool]>) -> Vec<SearchResult> {
        let query_tokens = tokenize(query);
        if query_tokens.is_empty() || self.doc_count == 0 {
            return Vec::new();
        }
        let (doc_count, avg_doc_len) = match excluded {
            None => (self.doc_count, self.avg_doc_len),
            Some(excluded) => self.served_corpus(excluded),
        };
        if doc_count == 0 {
            return Vec::new();
        }

        // Pre-allocated score array: O(1) per-access vs HashMap overhead.
        // Kolmogorov-optimal: minimal allocation for the scoring operation.
        let n = self.chunks.len();
        let mut scores = vec![0.0f64; n];
        let mut touched = Vec::with_capacity(n.min(256));

        for token in &query_tokens {
            let lower = token.to_lowercase();
            let Some(postings) = self.inverted.get(&lower) else {
                continue;
            };
            let df = match excluded {
                None => *self.doc_freqs.get(&lower).unwrap_or(&0),
                Some(excluded) => served_ranking::doc_freq(postings, excluded),
            };
            let df = df as f64;
            if df == 0.0 {
                continue;
            }

            let idf = ((doc_count as f64 - df + 0.5) / (df + 0.5) + 1.0).ln();

            for &(idx, weight) in postings {
                if excluded.is_some_and(|excluded| served_ranking::is_excluded(excluded, idx)) {
                    continue;
                }
                let doc_len = self.chunks[idx].token_count as f64;
                let norm_len = doc_len / avg_doc_len.max(1.0);
                let bm25 = idf * (weight * (BM25_K1 + 1.0))
                    / (weight + BM25_K1 * (1.0 - BM25_B + BM25_B * norm_len));

                if scores[idx] == 0.0 {
                    touched.push(idx);
                }
                scores[idx] += bm25;
            }
        }

        for &idx in &touched {
            scores[idx] += structural_query_boost(query, &self.chunks[idx]);
        }

        let mut results: Vec<SearchResult> = touched
            .iter()
            .filter(|&&idx| scores[idx] > 0.0)
            .map(|&idx| {
                let chunk = &self.chunks[idx];
                let snippet = query_snippet(&chunk.content, &query_tokens);
                SearchResult {
                    chunk_idx: idx,
                    score: scores[idx],
                    file_path: chunk.file_path.clone(),
                    symbol_name: chunk.symbol_name.clone(),
                    kind: chunk.kind.clone(),
                    start_line: chunk.start_line,
                    end_line: chunk.end_line,
                    snippet,
                }
            })
            .collect();

        results.sort_by(|a, b| {
            b.score
                .partial_cmp(&a.score)
                .unwrap_or(std::cmp::Ordering::Equal)
                .then_with(|| a.file_path.cmp(&b.file_path))
                .then_with(|| a.symbol_name.cmp(&b.symbol_name))
                .then_with(|| a.start_line.cmp(&b.start_line))
                .then_with(|| a.end_line.cmp(&b.end_line))
        });
        results
    }

    pub fn save(&self, root: &Path) -> std::io::Result<SaveOutcome> {
        if self.chunks.len() > CHUNK_COUNT_WARNING {
            tracing::warn!(
                "[bm25] index has {} chunks (threshold {}), consider adding extra_ignore_patterns",
                self.chunks.len(),
                CHUNK_COUNT_WARNING
            );
        }

        let dir = index_dir(root);
        std::fs::create_dir_all(&dir)?;

        // #790: stream postcard → zstd → file to avoid holding the full
        // serialized buffer in memory. Temp-file + rename for atomicity.
        let target = dir.join("bm25_index.bin.zst");
        let tmp = dir.join("bm25_index.bin.zst.tmp");
        {
            let file = std::fs::File::create(&tmp)?;
            let buf_writer = std::io::BufWriter::new(file);
            let mut encoder = zstd::Encoder::new(buf_writer, ZSTD_LEVEL)
                .map_err(|e| std::io::Error::other(format!("zstd encoder init: {e}")))?;
            postcard::to_io(self, &mut encoder)
                .map_err(|e| std::io::Error::other(e.to_string()))?;
            encoder.finish()?;
        }

        let compressed_bytes = std::fs::metadata(&tmp)?.len();
        let max_bytes = max_bm25_cache_bytes();
        if compressed_bytes > max_bytes {
            let _ = std::fs::remove_file(&tmp);
            tracing::warn!(
                "[bm25] compressed index too large ({:.1} MB, limit {:.0} MB), refusing to persist: {}",
                compressed_bytes as f64 / 1_048_576.0,
                max_bytes / (1024 * 1024),
                dir.display()
            );
            return Ok(SaveOutcome::SkippedTooLarge {
                compressed_bytes,
                limit_bytes: max_bytes,
            });
        }

        tracing::info!(
            "[bm25] index: {:.1} MB zstd compressed",
            compressed_bytes as f64 / 1_048_576.0,
        );

        std::fs::rename(&tmp, &target)?;

        let _ = std::fs::remove_file(dir.join("bm25_index.bin"));
        let _ = std::fs::remove_file(dir.join("bm25_index.json"));

        let _ = std::fs::write(
            dir.join("project_root.txt"),
            root.to_string_lossy().as_bytes(),
        );
        self.save_provider_origins(&dir);

        Ok(SaveOutcome::Persisted { compressed_bytes })
    }

    /// The persisted index, only when it was admitted under the current
    /// admission policy: chunks admitted under another policy (or a legacy
    /// index built from raw sources) are never served or reused (G5, E3).
    pub fn load(root: &Path) -> Option<Self> {
        let admission = crate::core::context_admission::stores::StoreAdmission::current();
        // The directory name is a hash of this checkout; the root recorded at
        // save is the authority. An index recorded for another root is
        // rebuilt instead of served. (A restored hosted bundle carries no
        // record; its directory is already this checkout's partition.)
        let canonical = |path: &Path| crate::core::pathutil::safe_canonicalize_bounded(path, 2000);
        if std::fs::read_to_string(index_dir(root).join("project_root.txt"))
            .is_ok_and(|stored| canonical(Path::new(stored.trim())) != canonical(root))
        {
            return None;
        }
        Self::load_persisted(root)
            .filter(|idx| idx.admission_policy.as_deref() == Some(admission.digest()))
            .map(|idx| idx.rooted_at(root).with_provider_origins(&index_dir(root)))
    }

    fn load_persisted(root: &Path) -> Option<Self> {
        let dir = index_dir(root);
        let max_bytes = max_bm25_cache_bytes();

        let zst_path = dir.join("bm25_index.bin.zst");
        if zst_path.exists() {
            let meta = std::fs::metadata(&zst_path).ok()?;
            if meta.len() > max_bytes {
                tracing::warn!(
                    "[bm25] compressed index too large ({:.1} GB, limit {:.0} MB), quarantining: {}",
                    meta.len() as f64 / 1_073_741_824.0,
                    max_bytes / (1024 * 1024),
                    zst_path.display()
                );
                let quarantined = zst_path.with_extension("zst.quarantined");
                let _ = std::fs::rename(&zst_path, &quarantined);
                return None;
            }
            let compressed = std::fs::read(&zst_path).ok()?;
            let max_decompressed = max_decompressed_bytes(meta.len());
            let data = bounded_zstd_decode(&compressed, max_decompressed)?;
            let idx: Self = postcard::from_bytes(&data).ok()?;
            return Some(idx);
        }

        let bin_path = dir.join("bm25_index.bin");
        if bin_path.exists() {
            let meta = std::fs::metadata(&bin_path).ok()?;
            if meta.len() > max_bytes {
                tracing::warn!(
                    "[bm25] index too large ({:.1} GB, limit {:.0} MB), quarantining: {}",
                    meta.len() as f64 / 1_073_741_824.0,
                    max_bytes / (1024 * 1024),
                    bin_path.display()
                );
                let quarantined = bin_path.with_extension("bin.quarantined");
                let _ = std::fs::rename(&bin_path, &quarantined);
                return None;
            }
            let data = std::fs::read(&bin_path).ok()?;
            let idx: Self = postcard::from_bytes(&data).ok()?;
            // Auto-migrate: compress legacy .bin to .bin.zst
            if let Ok(compressed) = zstd::encode_all(data.as_slice(), ZSTD_LEVEL) {
                let zst_tmp = zst_path.with_extension("zst.tmp");
                if std::fs::write(&zst_tmp, &compressed).is_ok()
                    && std::fs::rename(&zst_tmp, &zst_path).is_ok()
                {
                    tracing::info!(
                        "[bm25] migrated {:.1} MB → {:.1} MB zstd",
                        data.len() as f64 / 1_048_576.0,
                        compressed.len() as f64 / 1_048_576.0
                    );
                    let _ = std::fs::remove_file(&bin_path);
                }
            }
            return Some(idx);
        }

        let json_path = dir.join("bm25_index.json");
        if json_path.exists() {
            let meta = std::fs::metadata(&json_path).ok()?;
            if meta.len() > max_bytes {
                tracing::warn!(
                    "[bm25] index too large ({:.1} GB, limit {:.0} MB), quarantining: {}",
                    meta.len() as f64 / 1_073_741_824.0,
                    max_bytes / (1024 * 1024),
                    json_path.display()
                );
                let quarantined = json_path.with_extension("json.quarantined");
                let _ = std::fs::rename(&json_path, &quarantined);
                return None;
            }
            let data = std::fs::read_to_string(&json_path).ok()?;
            return serde_json::from_str(&data).ok();
        }

        None
    }

    pub fn load_or_build(root: &Path) -> Self {
        Self::load_or_build_inner(root, false)
    }

    /// Like `load_or_build` but uses a fast sentinel-sampling staleness check
    /// that skips the expensive full directory walk for new-file detection.
    pub fn load_or_build_fast(root: &Path) -> Self {
        Self::load_or_build_inner(root, true)
    }

    fn load_or_build_inner(root: &Path, fast_stale: bool) -> Self {
        if !is_safe_bm25_root(root) {
            return Self::default();
        }
        if let Some(idx) = Self::load(root) {
            let stale = if fast_stale {
                bm25_index_looks_stale_fast(&idx, root)
            } else {
                bm25_index_looks_stale(&idx, root)
            };
            if !stale {
                return idx;
            }
            tracing::debug!(
                "[bm25_index: stale index detected for {}; rebuilding]",
                root.display()
            );
            let rebuilt = if idx.files.is_empty() {
                Self::build_from_directory(root)
            } else {
                Self::rebuild_incremental(root, &idx)
            }
            .carrying_external_from(&idx);
            let _ = rebuilt.save(root);
            return rebuilt.rooted_at(root);
        }

        let built = Self::build_from_directory(root);
        let _ = built.save(root);
        built.rooted_at(root)
    }

    pub fn index_file_path(root: &Path) -> PathBuf {
        let dir = index_dir(root);
        let zst = dir.join("bm25_index.bin.zst");
        if zst.exists() {
            return zst;
        }
        let bin = dir.join("bm25_index.bin");
        if bin.exists() {
            return bin;
        }
        dir.join("bm25_index.json")
    }

    /// Chunk count of the persisted index without loading it (#1914): postcard
    /// writes `chunks` — the first field — as a varint length prefix, so only
    /// the first few decompressed bytes are read. `None` for the legacy JSON
    /// format or an unreadable file; `Some(0)` for an index that holds nothing.
    pub fn persisted_chunk_count(root: &Path) -> Option<u64> {
        use std::io::Read;
        let path = Self::index_file_path(root);
        let file = std::fs::File::open(&path).ok()?;
        let mut head: Box<dyn Read> = match path.extension().and_then(|e| e.to_str()) {
            Some("zst") => Box::new(zstd::Decoder::new(file).ok()?),
            Some("bin") => Box::new(file),
            _ => return None,
        };
        let mut count = 0u64;
        for shift in (0..64).step_by(7) {
            let mut byte = [0u8; 1];
            head.read_exact(&mut byte).ok()?;
            count |= u64::from(byte[0] & 0x7f) << shift;
            if byte[0] & 0x80 == 0 {
                return Some(count);
            }
        }
        None
    }
}

fn is_safe_bm25_root(root: &Path) -> bool {
    super::graph_index::is_safe_scan_root_public(&root.to_string_lossy())
}

fn bm25_index_looks_stale(index: &BM25Index, root: &Path) -> bool {
    bm25_index_looks_stale_inner(index, root, false)
}

/// Fast staleness check: samples a subset of tracked files and skips the
/// expensive `list_code_files()` walk for new-file detection.
pub fn bm25_index_looks_stale_fast(index: &BM25Index, root: &Path) -> bool {
    bm25_index_looks_stale_inner(index, root, true)
}

fn bm25_index_looks_stale_inner(index: &BM25Index, root: &Path, fast: bool) -> bool {
    if index.chunks.is_empty() {
        return false;
    }

    // An index persisted before secret-like paths were excluded from the walk
    // must be rebuilt, which drops them (GHSA-rxv5-4c3x-g4q6 A).
    if index
        .files
        .keys()
        .any(|rel| crate::core::io_boundary::is_secret_like(Path::new(rel)).is_some())
    {
        return true;
    }

    if index.files.is_empty() {
        let mut seen = std::collections::HashSet::<&str>::new();
        for chunk in &index.chunks {
            let rel = chunk.file_path.trim_start_matches(['/', '\\']);
            if rel.is_empty() {
                continue;
            }
            if !seen.insert(rel) {
                continue;
            }
            if !root.join(rel).exists() {
                return true;
            }
        }
        return false;
    }

    if fast {
        let sample_size = index.files.len().min(SENTINEL_SAMPLE_SIZE);
        let step = if index.files.len() > sample_size {
            index.files.len() / sample_size
        } else {
            1
        };
        for (i, (rel, old_state)) in index.files.iter().enumerate() {
            if i % step != 0 {
                continue;
            }
            let abs = root.join(rel);
            if !abs.exists() {
                return true;
            }
            let Some(cur) = IndexedFileState::from_path(&abs) else {
                return true;
            };
            if &cur != old_state {
                return true;
            }
        }
        // Sampling tracked files says nothing about files that were never
        // indexed, so without this the fast path serves a partial tree forever
        // (#1724). The directory snapshot closes that hole for one stat per
        // indexed directory.
        return dirs_look_stale(index, root);
    }

    for (rel, old_state) in &index.files {
        let abs = root.join(rel);
        if !abs.exists() {
            return true;
        }
        let Some(cur) = IndexedFileState::from_path(&abs) else {
            return true;
        };
        if &cur != old_state {
            return true;
        }
    }

    for rel in list_code_files(root) {
        if !index.files.contains_key(&rel) {
            return true;
        }
    }

    false
}

const SENTINEL_SAMPLE_SIZE: usize = 10;

/// True when recorded directory fingerprints no longer match the filesystem —
/// i.e. a file was added to (or removed from) the indexed tree.
///
/// An index persisted before directory snapshots existed carries no `dirs`, so
/// its freshness cannot be proven cheaply: it is reported stale exactly once,
/// and the resulting rebuild records the snapshot for every later call.
fn dirs_look_stale(index: &BM25Index, root: &Path) -> bool {
    if index.dirs.is_empty() {
        return true;
    }
    for (rel, recorded) in &index.dirs {
        let abs = if rel.is_empty() {
            root.to_path_buf()
        } else {
            root.join(rel)
        };
        if dir_fingerprint(&abs) != Some(*recorded) {
            return true;
        }
    }
    false
}

fn dir_fingerprint(path: &Path) -> Option<u64> {
    use std::hash::{Hash, Hasher};

    let meta = std::fs::metadata(path).ok()?;
    let modified = meta.modified().ok()?.duration_since(UNIX_EPOCH).ok()?;
    let mut entries = std::fs::read_dir(path)
        .ok()?
        .filter_map(Result::ok)
        .map(|entry| entry.file_name().to_string_lossy().into_owned())
        .collect::<Vec<_>>();
    entries.sort_unstable();
    let mut hasher = std::collections::hash_map::DefaultHasher::new();
    modified.as_nanos().hash(&mut hasher);
    entries.hash(&mut hasher);
    Some(hasher.finish())
}

/// Content snapshot of the root plus every ancestor directory of an indexed file.
/// Ancestors are included so a brand-new top-level directory (whose own mtime
/// was never recorded) is still caught through its recorded parent.
fn dir_states(root: &Path, files: &[String]) -> HashMap<String, u64> {
    let mut rels: std::collections::HashSet<String> = std::collections::HashSet::new();
    rels.insert(String::new());
    for rel in files {
        let mut cur = Path::new(rel);
        while let Some(parent) = cur.parent() {
            let key = parent.to_string_lossy().to_string();
            // Once a directory is known, so are all of its ancestors.
            if !rels.insert(key) {
                break;
            }
            cur = parent;
        }
    }
    rels.into_iter()
        .filter_map(|rel| {
            let abs = if rel.is_empty() {
                root.to_path_buf()
            } else {
                root.join(&rel)
            };
            dir_fingerprint(&abs).map(|fingerprint| (rel, fingerprint))
        })
        .collect()
}

/// Streams the persisted index out, refusing anything past `max_bytes`.
///
/// The buffer never grows beyond `max_bytes`, so the ceiling passed in *is* the
/// worst-case allocation — which is why #1739 hinged on that ceiling being a
/// sane number rather than on this loop. Our own writer streams postcard into
/// the encoder (#790) and so never records a frame content size, which rules
/// out a cheap header pre-check.
fn bounded_zstd_decode(compressed: &[u8], max_bytes: u64) -> Option<Vec<u8>> {
    use std::io::Read;
    let mut decoder = zstd::Decoder::new(compressed).ok()?;
    let mut buf = Vec::new();
    let mut chunk = vec![0u8; 65536];
    let mut total = 0u64;
    loop {
        let n = decoder.read(&mut chunk).ok()?;
        if n == 0 {
            break;
        }
        total += n as u64;
        if total > max_bytes {
            tracing::warn!(
                "[bm25] decompressed index exceeds limit ({:.0} MB > {:.0} MB), aborting load",
                total as f64 / (1024.0 * 1024.0),
                max_bytes as f64 / (1024.0 * 1024.0)
            );
            return None;
        }
        buf.extend_from_slice(&chunk[..n]);
    }
    Some(buf)
}

/// Worst-case RAM this process will spend decompressing a persisted index.
///
/// #1739: this was `max_bm25_cache_bytes() * 20` — twenty times a **disk**
/// budget, with no reference to the file at hand or to the machine's memory. On
/// the default 512 MB disk budget that authorised a 10 GB allocation for any
/// index, however small the file; because `ctx_search` reloads on demand,
/// repeated calls stacked to 20 GB RSS and froze the host.
///
/// The index is decompressed into RAM, so RAM is what has to bound it, and a
/// given file cannot plausibly expand past a fixed ratio of its own size.
fn max_decompressed_bytes(compressed_bytes: u64) -> u64 {
    /// postcard + zstd on an index this shape expands well under this.
    const MAX_EXPANSION_RATIO: u64 = 20;
    /// Keeps small indices loadable on hosts that report little or no RAM.
    const FLOOR: u64 = 64 * 1024 * 1024;

    let file_derived = compressed_bytes
        .saturating_mul(MAX_EXPANSION_RATIO)
        .max(FLOOR);
    match crate::core::memory_guard::rss_limit_bytes() {
        Some(limit) => file_derived.min(limit.max(FLOOR)),
        None => file_derived,
    }
}

fn index_dir(root: &Path) -> PathBuf {
    crate::core::index_namespace::vectors_dir(root)
}

/// Walk depth of the corpus walk. Deeper files are never indexed.
const CORPUS_MAX_DEPTH: usize = 20;

/// The rules that decide corpus membership, loaded once. Shared by
/// [`list_code_files`] and the `index why` diagnostic ([`coverage`]), so the
/// explanation can never drift from what the indexer actually does.
pub(crate) struct CorpusRules {
    filter: crate::core::index_filter::IndexFileFilter,
    /// `DEFAULT_BM25_IGNORES` followed by `extra_ignore_patterns`, with the
    /// source text kept so a diagnostic can name the pattern that matched.
    ignore_patterns: Vec<(String, glob::Pattern)>,
    max_files: usize,
}

impl CorpusRules {
    pub(crate) fn load() -> Self {
        Self::from_config(&crate::core::config::Config::load())
    }

    fn from_config(cfg: &crate::core::config::Config) -> Self {
        // #735: the declared corpus filter ([index] config + CLI overlay) decides
        // membership before anything is chunked; the semantic index chunks this
        // corpus, so it inherits the same universe.
        let filter = crate::core::index_filter::IndexFileFilter::resolve(cfg);
        let ignore_patterns = DEFAULT_BM25_IGNORES
            .iter()
            .map(|p| (*p).to_string())
            .chain(cfg.extra_ignore_patterns.iter().cloned())
            .filter_map(|p| glob::Pattern::new(&p).ok().map(|g| (p, g)))
            .collect();
        // Mirrors graph_index_max_files: 0 = unlimited, default 5000 (MAX_BM25_FILES).
        let max_files = if cfg.bm25_max_files == 0 {
            usize::MAX
        } else {
            cfg.bm25_max_files as usize
        };
        Self {
            filter,
            ignore_patterns,
            max_files,
        }
    }

    /// The corpus walker: gitignore per the filter, bounded depth. Callers add
    /// their own `filter_entry`.
    fn walker(&self, root: &Path) -> ignore::WalkBuilder {
        let mut builder = ignore::WalkBuilder::new(root);
        builder
            // #1792: the BM25 corpus is authoritative for `ctx_compose`/`ctx_overview`,
            // so omitting tracked dotfiles made "no match" indistinguishable from
            // "not indexed".
            .hidden(crate::core::walk_filter::SKIP_HIDDEN_IN_CONTENT_WALK)
            .git_ignore(self.filter.respect_gitignore)
            .git_global(self.filter.respect_gitignore)
            .git_exclude(self.filter.respect_gitignore)
            .require_git(false)
            .max_depth(Some(CORPUS_MAX_DEPTH));
        builder
    }

    /// First default/extra ignore pattern matching `rel`.
    fn ignore_pattern_for(&self, rel: &str) -> Option<&str> {
        self.ignore_patterns
            .iter()
            .find(|(_, p)| p.matches(rel))
            .map(|(src, _)| src.as_str())
    }

    /// Whether the declared `[index]` filter drops `rel`. Globs match on
    /// forward slashes so they behave identically on Windows; the index key
    /// keeps the platform separator (existing indexes stay valid).
    fn filter_excludes(&self, rel: &str) -> bool {
        self.filter.is_excluded(&rel.replace('\\', "/"))
    }
}