lean_ctx/core/bm25_index/
coverage.rs

1// SPDX-License-Identifier: Apache-2.0
2//! `lean-ctx index why <file>`: explain whether a file is in the search corpus
3//! and, if not, which rule dropped it.
4//!
5//! An incomplete index is indistinguishable from an empty result — `ctx_compose`
6//! just says "no match" — so every stage that can drop a file is replayed here
7//! with the indexer's own rules (`CorpusRules`, the same walk, the same
8//! size/binary/minified checks) in the order the indexer applies them. The
9//! first stage that rejects the file is the answer.
10
11use std::path::{Path, PathBuf};
12use std::sync::{Arc, Mutex};
13
14use serde::Serialize;
15
16use super::{
17    BM25Index, CORPUS_MAX_DEPTH, CorpusRules, IndexedFileState, build::MAX_FILE_SIZE_BYTES,
18    list_code_files, looks_minified,
19};
20use crate::core::ingestion::IngestKind;
21
22/// Why a file is not part of the corpus. Variants are ordered like the
23/// indexer's stages.
24#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
25#[serde(tag = "reason", rename_all = "snake_case")]
26pub enum Exclusion {
27    NotFound,
28    NotAFile,
29    OutsideRoot,
30    /// A directory on the path is pruned by the walk (dependency/vendor
31    /// directory, agent worktree copy) or the file is a cloud placeholder.
32    PrunedDirectory {
33        path: String,
34        kind: &'static str,
35    },
36    /// Ignored by `.gitignore`, `.ignore`, `.git/info/exclude` or the global
37    /// gitignore.
38    Gitignored,
39    TooDeep {
40        depth: usize,
41        limit: usize,
42    },
43    /// Symlinked directories are not followed by the corpus walk.
44    NotWalked,
45    Binary,
46    Lockfile,
47    IgnorePattern {
48        pattern: String,
49    },
50    IndexFilter {
51        summary: String,
52    },
53    FileCap {
54        cap: usize,
55    },
56    TooLarge {
57        size_bytes: u64,
58        limit_bytes: u64,
59    },
60    BinaryContent,
61    NoExtractableText,
62    Unreadable {
63        error: String,
64    },
65    Minified,
66    /// The context gateway does not admit the file into derived stores
67    /// (restricted or withheld, G5); it is deliberately absent.
68    WithheldByGateway,
69}
70
71impl Exclusion {
72    /// One line: what dropped the file.
73    #[must_use]
74    pub(crate) fn describe(&self) -> String {
75        match self {
76            Self::NotFound => "file does not exist".into(),
77            Self::NotAFile => "not a regular file".into(),
78            Self::OutsideRoot => "outside the project root".into(),
79            Self::PrunedDirectory { path, kind } => format!("inside {path}/ — {kind}"),
80            Self::Gitignored => {
81                "ignored by .gitignore / .ignore / .git/info/exclude / global gitignore".into()
82            }
83            Self::TooDeep { depth, limit } => {
84                format!("nested {depth} levels deep; the corpus walk stops at {limit}")
85            }
86            Self::NotWalked => {
87                "not reachable by the corpus walk (symlinked directories are not followed)".into()
88            }
89            Self::Binary => "binary file type".into(),
90            Self::Lockfile => "dependency lockfile (generated content)".into(),
91            Self::IgnorePattern { pattern } => format!("matches ignore pattern `{pattern}`"),
92            Self::IndexFilter { summary } => format!("excluded by the [index] filter ({summary})"),
93            Self::FileCap { cap } => {
94                format!("file cap reached: the corpus is limited to {cap} files")
95            }
96            Self::TooLarge {
97                size_bytes,
98                limit_bytes,
99            } => format!(
100                "{} exceeds the per-file limit of {}",
101                human_bytes(*size_bytes),
102                human_bytes(*limit_bytes)
103            ),
104            Self::BinaryContent => "content is binary (NUL bytes in the first 8 KiB)".into(),
105            Self::NoExtractableText => "document yielded no extractable text".into(),
106            Self::Unreadable { error } => format!("cannot be read: {error}"),
107            Self::Minified => "looks minified/bundled (very long lines)".into(),
108            Self::WithheldByGateway => {
109                "withheld by the context gateway (restricted or not admissible); \
110                 never stored in an index"
111                    .into()
112            }
113        }
114    }
115
116    /// How to get the file indexed, when the user can change it.
117    #[must_use]
118    pub(crate) fn remedy(&self) -> Option<String> {
119        let remedy = match self {
120            Self::OutsideRoot => "pass --root <project> to diagnose another project",
121            Self::Gitignored => {
122                "lean-ctx index build --no-gitignore, or [index] respect_gitignore = false"
123            }
124            Self::IgnorePattern { .. } => {
125                "built-in patterns are fixed; for extra_ignore_patterns edit the config"
126            }
127            Self::IndexFilter { .. } => {
128                "adjust [index] include/exclude or the --include/--exclude flags"
129            }
130            Self::FileCap { .. } => "set bm25_max_files in config (0 = unlimited)",
131            Self::PrunedDirectory { .. }
132            | Self::NotWalked
133            | Self::NotFound
134            | Self::NotAFile
135            | Self::TooDeep { .. }
136            | Self::Binary
137            | Self::Lockfile
138            | Self::TooLarge { .. }
139            | Self::BinaryContent
140            | Self::NoExtractableText
141            | Self::Unreadable { .. }
142            | Self::Minified => return None,
143            Self::WithheldByGateway => {
144                "lean-ctx inspect explains the decision; ctx_read shows what may be read"
145            }
146        };
147        Some(remedy.to_string())
148    }
149}
150
151/// What the persisted BM25 index holds for an eligible file.
152#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
153#[serde(tag = "state", rename_all = "snake_case")]
154pub enum IndexState {
155    /// No BM25 index has been built (or it could not be loaded).
156    NoIndex,
157    /// Eligible but absent from the index — built before the file existed, or
158    /// by an older version that could not read its encoding.
159    Missing,
160    /// Indexed; `stale` when the file changed since.
161    Indexed { chunks: usize, stale: bool },
162}
163
164#[derive(Debug, Clone, Serialize)]
165pub struct FileCoverage {
166    /// Path relative to the project root (as the index keys it), or the input
167    /// when it could not be resolved.
168    pub path: String,
169    pub kind: Option<&'static str>,
170    pub encoding: Option<&'static str>,
171    pub size_bytes: Option<u64>,
172    pub exclusion: Option<Exclusion>,
173    /// Only set when the file is eligible.
174    pub index: Option<IndexState>,
175}
176
177impl FileCoverage {
178    fn excluded(path: String, exclusion: Exclusion) -> Self {
179        Self {
180            path,
181            kind: None,
182            encoding: None,
183            size_bytes: None,
184            exclusion: Some(exclusion),
185            index: None,
186        }
187    }
188}
189
190/// Resolve `input` (absolute, relative to `root`, or relative to the cwd —
191/// in that order, since index paths are root-relative) and explain its corpus
192/// membership.
193#[must_use]
194pub fn explain(root: &Path, input: &str) -> FileCoverage {
195    let candidate = Path::new(input);
196    let abs = if candidate.is_absolute() {
197        candidate.to_path_buf()
198    } else if root.join(candidate).exists() {
199        root.join(candidate)
200    } else {
201        std::env::current_dir().map_or_else(|_| root.join(candidate), |cwd| cwd.join(candidate))
202    };
203    let Ok(abs) = abs.canonicalize() else {
204        return FileCoverage::excluded(input.to_string(), Exclusion::NotFound);
205    };
206    let Some(rel) = root
207        .canonicalize()
208        .ok()
209        .and_then(|r| abs.strip_prefix(r).ok().map(Path::to_path_buf))
210    else {
211        return FileCoverage::excluded(input.to_string(), Exclusion::OutsideRoot);
212    };
213    let rel = rel.to_string_lossy().to_string();
214    if !abs.is_file() {
215        return FileCoverage::excluded(rel, Exclusion::NotAFile);
216    }
217    explain_rel(root, &rel)
218}
219
220/// Explain an existing file given as `rel` (platform separators) under `root`.
221fn explain_rel(root: &Path, rel: &str) -> FileCoverage {
222    let rules = CorpusRules::load();
223    // Walk paths are `root` + components, uncanonicalized — the same spelling
224    // the corpus walk produces.
225    let target = root.join(rel);
226    let mut out = FileCoverage::excluded(rel.to_string(), Exclusion::NotFound);
227    out.exclusion = None;
228    out.size_bytes = target.metadata().ok().map(|m| m.len());
229    if let Ok(bytes) = read_prefix(&target) {
230        out.encoding = Some(if crate::core::text_decode::looks_binary(&bytes) {
231            "binary"
232        } else {
233            crate::core::text_decode::detect_encoding(&bytes).label()
234        });
235    }
236
237    if let Some(exclusion) = walk_exclusion(&rules, root, &target, rel) {
238        out.exclusion = Some(exclusion);
239        return out;
240    }
241
242    let kind = crate::core::ingestion::classify_path(&target);
243    out.kind = Some(kind.as_str());
244    let exclusion = match kind {
245        IngestKind::Binary => Some(Exclusion::Binary),
246        IngestKind::Generated => Some(Exclusion::Lockfile),
247        _ => None,
248    }
249    .or_else(|| {
250        rules
251            .ignore_pattern_for(rel)
252            .map(|pattern| Exclusion::IgnorePattern {
253                pattern: pattern.to_string(),
254            })
255    })
256    .or_else(|| {
257        rules.filter_excludes(rel).then(|| Exclusion::IndexFilter {
258            summary: rules.filter.summary().unwrap_or_default(),
259        })
260    })
261    .or_else(|| content_exclusion(&target))
262    .or_else(|| {
263        // Every per-file rule admits it; only the corpus-wide cap is left.
264        let corpus = list_code_files(root);
265        corpus.binary_search(&rel.to_string()).is_err().then_some(
266            if corpus.len() >= rules.max_files {
267                Exclusion::FileCap {
268                    cap: rules.max_files,
269                }
270            } else {
271                Exclusion::NotWalked
272            },
273        )
274    });
275    if exclusion.is_some() {
276        out.exclusion = exclusion;
277        return out;
278    }
279
280    out.index = Some(index_state(root, rel, &target));
281    out
282}
283
284/// Replays the corpus walk restricted to the target's ancestor chain, so the
285/// cost is proportional to the path depth, not to the repository. A second,
286/// unfiltered walk tells gitignore apart from pruning and depth.
287fn walk_exclusion(rules: &CorpusRules, root: &Path, target: &Path, rel: &str) -> Option<Exclusion> {
288    let on_path = {
289        let target = target.to_path_buf();
290        move |entry: &ignore::DirEntry| target.starts_with(entry.path())
291    };
292
293    let corpus_walk = {
294        let on_path = on_path.clone();
295        rules
296            .walker(root)
297            .filter_entry(move |e| on_path(e) && crate::core::walk_filter::keep_entry(e))
298            .build()
299    };
300    if reaches(corpus_walk, target) {
301        return None;
302    }
303
304    let pruned: Arc<Mutex<Option<(PathBuf, &'static str)>>> = Arc::new(Mutex::new(None));
305    let unfiltered_walk = {
306        let pruned = Arc::clone(&pruned);
307        ignore::WalkBuilder::new(root)
308            .standard_filters(false)
309            .filter_entry(move |e| {
310                if !on_path(e) {
311                    return false;
312                }
313                let kind = prune_kind(e);
314                if let Some(kind) = kind
315                    && let Ok(mut slot) = pruned.lock()
316                {
317                    slot.get_or_insert_with(|| (e.path().to_path_buf(), kind));
318                }
319                kind.is_none()
320            })
321            .build()
322    };
323    let reachable = reaches(unfiltered_walk, target);
324    let pruned = pruned.lock().ok().and_then(|mut slot| slot.take());
325    if let Some((dir, kind)) = pruned {
326        let shown = dir
327            .strip_prefix(root)
328            .unwrap_or(&dir)
329            .to_string_lossy()
330            .to_string();
331        return Some(Exclusion::PrunedDirectory { path: shown, kind });
332    }
333    if !reachable {
334        return Some(Exclusion::NotWalked);
335    }
336    let depth = Path::new(rel).components().count();
337    if depth > CORPUS_MAX_DEPTH {
338        return Some(Exclusion::TooDeep {
339            depth,
340            limit: CORPUS_MAX_DEPTH,
341        });
342    }
343    Some(Exclusion::Gitignored)
344}
345
346fn reaches(walk: ignore::Walk, target: &Path) -> bool {
347    walk.flatten().any(|e| e.path() == target)
348}
349
350/// Mirrors [`crate::core::walk_filter::keep_entry`], naming the rule.
351fn prune_kind(entry: &ignore::DirEntry) -> Option<&'static str> {
352    if crate::core::walk_filter::is_vcs_metadata(entry) {
353        Some("version-control metadata, never indexed")
354    } else if crate::core::walk_filter::is_vendor_dir(entry) {
355        Some("dependency/vendor directory, never indexed")
356    } else if crate::core::walk_filter::is_agent_worktree_dir(entry) {
357        Some("agent worktree copy, never indexed")
358    } else if crate::core::walk_filter::is_linked_git_worktree(entry) {
359        Some("linked git worktree (a second checkout of the repository), never indexed")
360    } else if !crate::core::cloud_files::keep_entry(entry) {
361        Some("cloud placeholder that is not downloaded")
362    } else {
363        None
364    }
365}
366
367/// The build's per-file content checks, in `prepare_file` order.
368fn content_exclusion(target: &Path) -> Option<Exclusion> {
369    let size_bytes = target.metadata().map(|m| m.len()).unwrap_or(0);
370    if size_bytes > MAX_FILE_SIZE_BYTES {
371        return Some(Exclusion::TooLarge {
372            size_bytes,
373            limit_bytes: MAX_FILE_SIZE_BYTES,
374        });
375    }
376    let content = if crate::core::extractors::is_binary_document(target) {
377        match std::fs::read(target) {
378            Ok(bytes) => {
379                let text = crate::core::extractors::extract(target, &bytes).text;
380                if text.is_empty() {
381                    return Some(Exclusion::NoExtractableText);
382                }
383                text
384            }
385            Err(e) => {
386                return Some(Exclusion::Unreadable {
387                    error: e.to_string(),
388                });
389            }
390        }
391    } else {
392        match crate::core::text_decode::read_text(target) {
393            Ok(text) => text,
394            Err(e) if e.kind() == std::io::ErrorKind::InvalidData => {
395                return Some(Exclusion::BinaryContent);
396            }
397            Err(e) => {
398                return Some(Exclusion::Unreadable {
399                    error: e.to_string(),
400                });
401            }
402        }
403    };
404    if looks_minified(&content) {
405        return Some(Exclusion::Minified);
406    }
407    // Same store admission the build applies (G5): a file the gateway keeps
408    // out of every index must be explained as such, not reported as missing.
409    crate::core::context_admission::stores::StoreAdmission::current()
410        .admit(&content, target, None)
411        .is_none()
412        .then_some(Exclusion::WithheldByGateway)
413}
414
415fn index_state(root: &Path, rel: &str, target: &Path) -> IndexState {
416    let Some(index) = BM25Index::load(root) else {
417        return IndexState::NoIndex;
418    };
419    let Some(stored) = index.files.get(rel) else {
420        return IndexState::Missing;
421    };
422    IndexState::Indexed {
423        chunks: index.chunks.iter().filter(|c| c.file_path == rel).count(),
424        stale: IndexedFileState::from_path(target).as_ref() != Some(stored),
425    }
426}
427
428/// Human-readable verdict (CLI `index why` and `ctx_index action=why`): the
429/// verdict first, then the remedy. Deterministic for a given file state.
430#[must_use]
431pub fn render(cov: &FileCoverage) -> String {
432    use std::fmt::Write as _;
433
434    let mut out = String::new();
435    let _ = writeln!(out, "  File:      {}", cov.path);
436    if let Some(kind) = cov.kind {
437        let _ = writeln!(out, "  Kind:      {kind}");
438    }
439    if let Some(encoding) = cov.encoding {
440        let _ = writeln!(out, "  Encoding:  {encoding}");
441    }
442    if let Some(exclusion) = &cov.exclusion {
443        let _ = writeln!(out, "  Corpus:    not indexed — {}", exclusion.describe());
444        if let Some(remedy) = exclusion.remedy() {
445            let _ = writeln!(out, "  Fix:       {remedy}");
446        }
447        return out;
448    }
449    let _ = writeln!(
450        out,
451        "  Corpus:    eligible (BM25 and the semantic index chunk this corpus)"
452    );
453    let bm25 = match &cov.index {
454        Some(IndexState::Indexed {
455            chunks,
456            stale: false,
457        }) => {
458            format!("indexed, {chunks} chunks, up to date")
459        }
460        Some(IndexState::Indexed {
461            chunks,
462            stale: true,
463        }) => format!(
464            "indexed, {chunks} chunks, changed since the last build — run: lean-ctx index build"
465        ),
466        Some(IndexState::Missing) => "missing from the current index (built before this file, \
467             or by an older version that could not read its encoding) — run: lean-ctx index build"
468            .to_string(),
469        Some(IndexState::NoIndex) | None => {
470            "no index built yet — run: lean-ctx index build".to_string()
471        }
472    };
473    let _ = writeln!(out, "  BM25:      {bm25}");
474    out
475}
476
477fn read_prefix(path: &Path) -> std::io::Result<Vec<u8>> {
478    use std::io::Read;
479    let mut buf = Vec::with_capacity(8192);
480    std::fs::File::open(path)?
481        .take(64 * 1024)
482        .read_to_end(&mut buf)?;
483    Ok(buf)
484}
485
486fn human_bytes(bytes: u64) -> String {
487    const MIB: u64 = 1024 * 1024;
488    if bytes >= MIB {
489        format!("{:.1} MiB", bytes as f64 / MIB as f64)
490    } else {
491        format!("{:.0} KiB", (bytes as f64 / 1024.0).ceil())
492    }
493}
494
495#[cfg(test)]
496mod tests {
497    use super::*;
498
499    /// A git project plus an isolated data dir, so the user's config (extra
500    /// ignore patterns, file cap, `[index]` filter) and indexes stay out.
501    struct Project {
502        dir: tempfile::TempDir,
503        _data: tempfile::TempDir,
504        _env: crate::core::data_dir::TestEnvGuard,
505    }
506
507    impl Project {
508        fn path(&self) -> &Path {
509            self.dir.path()
510        }
511    }
512
513    impl Drop for Project {
514        fn drop(&mut self) {
515            crate::test_env::remove_var("LEAN_CTX_DATA_DIR");
516        }
517    }
518
519    fn project() -> Project {
520        let env = crate::core::data_dir::test_env_lock();
521        let data = tempfile::tempdir().expect("data dir");
522        crate::test_env::set_var("LEAN_CTX_DATA_DIR", data.path());
523        let dir = tempfile::tempdir().expect("tempdir");
524        std::fs::create_dir_all(dir.path().join(".git")).expect("git dir");
525        Project {
526            dir,
527            _data: data,
528            _env: env,
529        }
530    }
531
532    fn write(root: &Path, rel: &str, bytes: &[u8]) {
533        let path = root.join(rel);
534        std::fs::create_dir_all(path.parent().expect("parent")).expect("mkdir");
535        std::fs::write(path, bytes).expect("write");
536    }
537
538    fn reason(root: &Path, rel: &str) -> Option<Exclusion> {
539        explain(root, rel).exclusion
540    }
541
542    #[test]
543    fn eligible_windows_1252_file_is_admitted_and_named() {
544        let dir = project();
545        write(
546            dir.path(),
547            "src/Legacy.cs",
548            b"// Gr\xF6\xDFe\r\nclass Legacy {}\r\n",
549        );
550        let cov = explain(dir.path(), "src/Legacy.cs");
551        assert_eq!(cov.exclusion, None, "{cov:?}");
552        assert_eq!(cov.kind, Some("code"));
553        assert_eq!(cov.encoding, Some("windows-1252 (legacy ANSI)"));
554        assert!(matches!(
555            cov.index,
556            Some(IndexState::NoIndex | IndexState::Missing)
557        ));
558    }
559
560    #[test]
561    fn missing_and_outside_root_are_reported() {
562        let dir = project();
563        assert_eq!(reason(dir.path(), "nope.rs"), Some(Exclusion::NotFound));
564        let other = tempfile::tempdir().expect("tempdir");
565        write(other.path(), "x.rs", b"fn x() {}");
566        let outside = other.path().join("x.rs");
567        assert_eq!(
568            reason(dir.path(), outside.to_str().expect("utf8 path")),
569            Some(Exclusion::OutsideRoot)
570        );
571        std::fs::create_dir_all(dir.path().join("src")).expect("mkdir");
572        assert_eq!(reason(dir.path(), "src"), Some(Exclusion::NotAFile));
573    }
574
575    #[test]
576    fn gitignored_file_is_reported() {
577        let dir = project();
578        write(dir.path(), ".gitignore", b"generated/\n");
579        write(dir.path(), "generated/api.rs", b"fn api() {}");
580        assert_eq!(
581            reason(dir.path(), "generated/api.rs"),
582            Some(Exclusion::Gitignored)
583        );
584    }
585
586    #[test]
587    fn vendor_directory_is_named() {
588        let dir = project();
589        write(
590            dir.path(),
591            "node_modules/lib/index.js",
592            b"module.exports = 1;",
593        );
594        assert_eq!(
595            reason(dir.path(), "node_modules/lib/index.js"),
596            Some(Exclusion::PrunedDirectory {
597                path: "node_modules".into(),
598                kind: "dependency/vendor directory, never indexed",
599            })
600        );
601    }
602
603    #[test]
604    fn default_ignore_pattern_is_named() {
605        let dir = project();
606        write(dir.path(), "app.min.js", b"var a=1;");
607        assert_eq!(
608            reason(dir.path(), "app.min.js"),
609            Some(Exclusion::IgnorePattern {
610                pattern: "*.min.js".into()
611            })
612        );
613    }
614
615    #[test]
616    fn binary_types_lockfiles_and_binary_content_are_distinguished() {
617        let dir = project();
618        write(dir.path(), "logo.png", b"\x89PNG\r\n\x1a\n");
619        write(dir.path(), "Cargo.lock", b"# lock\n");
620        write(dir.path(), "data.cs", b"class A {}\0\0\0");
621        assert_eq!(reason(dir.path(), "logo.png"), Some(Exclusion::Binary));
622        assert_eq!(reason(dir.path(), "Cargo.lock"), Some(Exclusion::Lockfile));
623        assert_eq!(
624            reason(dir.path(), "data.cs"),
625            Some(Exclusion::BinaryContent)
626        );
627    }
628
629    #[test]
630    fn utf16_file_is_eligible() {
631        let dir = project();
632        let mut bytes = vec![0xFF, 0xFE];
633        for unit in "class Wide {}\r\n".encode_utf16() {
634            bytes.extend_from_slice(&unit.to_le_bytes());
635        }
636        write(dir.path(), "Wide.cs", &bytes);
637        let cov = explain(dir.path(), "Wide.cs");
638        assert_eq!(cov.exclusion, None, "{cov:?}");
639        assert_eq!(cov.encoding, Some("utf-16le (BOM)"));
640    }
641
642    #[test]
643    fn oversized_and_minified_files_are_reported() {
644        let dir = project();
645        let big = vec![b'a'; usize::try_from(MAX_FILE_SIZE_BYTES).expect("fits") + 1];
646        write(dir.path(), "huge.txt", &big);
647        assert!(matches!(
648            reason(dir.path(), "huge.txt"),
649            Some(Exclusion::TooLarge { .. })
650        ));
651        let minified = format!("var x={};\n", "1+".repeat(40_000));
652        write(dir.path(), "bundle.js", minified.as_bytes());
653        assert_eq!(reason(dir.path(), "bundle.js"), Some(Exclusion::Minified));
654    }
655
656    /// G5: a file the gateway keeps out of every index is explained as such —
657    /// not reported as indexable and then silently missing.
658    #[test]
659    fn a_file_withheld_by_the_gateway_is_explained() {
660        let dir = project();
661        write(
662            dir.path(),
663            "plans.md",
664            b"*** TOP SECRET ***\nlaunch window\n",
665        );
666        write(dir.path(), "notes.md", b"plain release notes\n");
667        assert_eq!(
668            reason(dir.path(), "plans.md"),
669            Some(Exclusion::WithheldByGateway)
670        );
671        assert_eq!(reason(dir.path(), "notes.md"), None);
672        assert!(Exclusion::WithheldByGateway.remedy().is_some());
673    }
674
675    #[test]
676    fn built_index_reports_chunks_and_staleness() {
677        let dir = project();
678        write(
679            dir.path(),
680            "src/lib.rs",
681            b"pub fn alpha() {}\n\npub fn beta() {}\n",
682        );
683        let index = BM25Index::build_from_directory(dir.path());
684        index.save(dir.path()).expect("save index");
685        match explain(dir.path(), "src/lib.rs").index {
686            Some(IndexState::Indexed { chunks, stale }) => {
687                assert!(chunks > 0);
688                assert!(!stale);
689            }
690            other => panic!("expected indexed, got {other:?}"),
691        }
692        write(dir.path(), "src/new.rs", b"pub fn gamma() {}\n");
693        assert_eq!(
694            explain(dir.path(), "src/new.rs").index,
695            Some(IndexState::Missing)
696        );
697    }
698
699    /// Regression for the Windows report: ANSI and UTF-16 files used to be
700    /// dropped by `read_to_string` and were unsearchable.
701    #[test]
702    fn legacy_encoded_files_are_indexed_and_searchable() {
703        let dir = project();
704        write(
705            dir.path(),
706            "src/Kuendigung.cs",
707            b"class Kuendigung\r\n{\r\n    // K\xFCndigungsfrist berechnen\r\n    int Tage() { return 30; }\r\n}\r\n",
708        );
709        let mut utf16 = vec![0xFF, 0xFE];
710        for unit in "class Storno\r\n{\r\n    // Stornogeb\u{fc}hr\r\n    int Betrag() { return 5; }\r\n}\r\n".encode_utf16() {
711            utf16.extend_from_slice(&unit.to_le_bytes());
712        }
713        write(dir.path(), "src/Storno.cs", &utf16);
714
715        let index = BM25Index::build_from_directory(dir.path());
716        index.save(dir.path()).expect("save index");
717
718        for (file, query) in [
719            ("src/Kuendigung.cs", "Kündigungsfrist"),
720            ("src/Storno.cs", "Stornogebühr"),
721        ] {
722            assert!(
723                matches!(
724                    explain(dir.path(), file).index,
725                    Some(IndexState::Indexed { chunks, .. }) if chunks > 0
726                ),
727                "{file} must be indexed"
728            );
729            let hits = index.search(query, 5);
730            assert!(
731                hits.iter().any(|h| h.file_path.replace('\\', "/") == file),
732                "{query} must find {file}, got {hits:?}"
733            );
734        }
735    }
736
737    #[test]
738    fn render_leads_with_the_verdict_and_remedy() {
739        let p = project();
740        write(p.path(), ".gitignore", b"out/\n");
741        write(p.path(), "out/gen.rs", b"fn g() {}");
742        let text = render(&explain(p.path(), "out/gen.rs"));
743        assert!(
744            text.contains("Corpus:    not indexed — ignored by .gitignore"),
745            "{text}"
746        );
747        assert!(
748            text.contains("Fix:       lean-ctx index build --no-gitignore"),
749            "{text}"
750        );
751
752        write(p.path(), "src/ok.rs", b"fn ok() {}");
753        let text = render(&explain(p.path(), "src/ok.rs"));
754        assert!(text.contains("Corpus:    eligible"), "{text}");
755        assert!(text.contains("BM25:      no index built yet"), "{text}");
756    }
757
758    #[test]
759    fn every_exclusion_describes_itself() {
760        let all = [
761            Exclusion::NotFound,
762            Exclusion::Gitignored,
763            Exclusion::FileCap { cap: 5000 },
764            Exclusion::TooLarge {
765                size_bytes: 3 * 1024 * 1024,
766                limit_bytes: MAX_FILE_SIZE_BYTES,
767            },
768        ];
769        for e in all {
770            assert!(!e.describe().is_empty(), "`e.describe()` is empty");
771        }
772        assert_eq!(
773            Exclusion::TooLarge {
774                size_bytes: 3 * 1024 * 1024,
775                limit_bytes: MAX_FILE_SIZE_BYTES
776            }
777            .describe(),
778            "3.0 MiB exceeds the per-file limit of 2.0 MiB"
779        );
780        assert!(Exclusion::FileCap { cap: 1 }.remedy().is_some());
781        assert!(Exclusion::Binary.remedy().is_none());
782    }
783}
lean_ctx/core/bm25_index/coverage.rs

lean_ctx/core/bm25_index/
coverage.rs
