lean_ctx/core/
ingestion.rs

1//! Ingestion front-door (`ingestion-spec-v1`).
2//!
3//! Decides what reaches the index — BM25, semantic, knowledge — **independent of
4//! code-ness**. Historically only files passing `is_code_file` were indexed,
5//! which locked lean-ctx to source repositories. Intake is now driven by a
6//! content-*kind* classification (extension fast-path + a bounded binary sniff),
7//! so any text corpus (docs, data, transcripts, logs) is ingestible while
8//! genuine binaries are excluded. Code repositories behave exactly as before:
9//! every kind that used to index still indexes.
10
11use std::path::Path;
12
13/// What kind of content a path holds, for intake decisions.
14#[derive(Debug, Clone, Copy, PartialEq, Eq)]
15pub enum IngestKind {
16    /// Source code — eligible for AST/symbol-aware processing downstream.
17    Code,
18    /// Prose / human documents (markdown, txt, html, email, …).
19    Document,
20    /// Structured data (json, yaml, toml, csv, xml, …).
21    Data,
22    /// Other UTF-8 text (unknown extension but verified textual).
23    Text,
24    /// Not ingestible as text (images, media, archives, binary documents).
25    Binary,
26}
27
28impl IngestKind {
29    /// Whether content of this kind should be fed to the index.
30    #[must_use]
31    pub fn is_ingestible(self) -> bool {
32        !matches!(self, IngestKind::Binary)
33    }
34
35    /// Stable lowercase label (for capabilities / diagnostics).
36    #[must_use]
37    pub fn as_str(self) -> &'static str {
38        match self {
39            IngestKind::Code => "code",
40            IngestKind::Document => "document",
41            IngestKind::Data => "data",
42            IngestKind::Text => "text",
43            IngestKind::Binary => "binary",
44        }
45    }
46}
47
48/// Prose / document extensions.
49const DOCUMENT_EXTS: &[&str] = &[
50    "md", "markdown", "mdx", "txt", "text", "rst", "org", "adoc", "asciidoc", "tex", "html", "htm",
51    "xhtml", "eml", "mbox", "log", "srt", "vtt",
52];
53
54/// Structured-data extensions.
55const DATA_EXTS: &[&str] = &[
56    "json",
57    "jsonl",
58    "ndjson",
59    "yaml",
60    "yml",
61    "toml",
62    "csv",
63    "tsv",
64    "xml",
65    "ini",
66    "cfg",
67    "conf",
68    "properties",
69    "graphql",
70    "proto",
71];
72
73/// Extensions we know are binary or not useful as raw text. Binary documents
74/// that *do* have a dedicated extractor (currently PDF) live in
75/// [`EXTRACTABLE_DOC_EXTS`] instead and are ingestible; office formats without
76/// an extractor yet (DOCX, XLSX, …) stay here and are skipped.
77const BINARY_EXTS: &[&str] = &[
78    // images
79    "png", "jpg", "jpeg", "gif", "webp", "bmp", "ico", "tif", "tiff", "heic", "avif", //
80    // media
81    "mp3", "wav", "flac", "ogg", "mp4", "mov", "avi", "mkv", "webm", //
82    // archives / packages
83    "zip", "gz", "tgz", "bz2", "xz", "zst", "7z", "rar", "tar", "jar", "war", //
84    // binary docs without an extractor yet (PDF lives in EXTRACTABLE_DOC_EXTS)
85    "doc", "docx", "xls", "xlsx", "ppt", "pptx", "odt", //
86    // compiled / binary artifacts
87    "exe", "dll", "so", "dylib", "o", "a", "class", "wasm", "bin", "dat", //
88    // fonts / db / images-vector-binary
89    "ttf", "otf", "woff", "woff2", "db", "sqlite", "lock",
90];
91
92/// Binary document formats that have a dedicated byte-level extractor
93/// ([`super::extractors`]) and are therefore ingestible: the indexer reads their
94/// raw bytes and converts them to text at index time rather than skipping them
95/// as opaque binaries. Grows as binary extractors (DOCX, XLSX, …) are added.
96const EXTRACTABLE_DOC_EXTS: &[&str] = &["pdf"];
97
98/// Max bytes inspected when sniffing an unknown-extension file.
99const SNIFF_BYTES: usize = 8192;
100
101/// Classify a path into an [`IngestKind`].
102///
103/// Fast path is extension-based; files with an unknown extension are sniffed
104/// (bounded read) so textual content is still picked up and binaries rejected.
105#[must_use]
106pub fn classify_path(path: &Path) -> IngestKind {
107    let ext = path
108        .extension()
109        .and_then(|e| e.to_str())
110        .unwrap_or("")
111        .to_lowercase();
112
113    if BINARY_EXTS.contains(&ext.as_str()) {
114        return IngestKind::Binary;
115    }
116    // Binary documents with a dedicated extractor (PDF, …): ingestible as
117    // documents — the indexer routes their bytes through `extractors::extract`
118    // instead of reading them as UTF-8.
119    if EXTRACTABLE_DOC_EXTS.contains(&ext.as_str()) {
120        return IngestKind::Document;
121    }
122    if crate::core::bm25_index::is_code_file(path) {
123        return IngestKind::Code;
124    }
125    if DOCUMENT_EXTS.contains(&ext.as_str()) {
126        return IngestKind::Document;
127    }
128    if DATA_EXTS.contains(&ext.as_str()) {
129        return IngestKind::Data;
130    }
131    // Unknown extension (or none): verify it is actually text before ingesting.
132    if looks_textual(path) {
133        IngestKind::Text
134    } else {
135        IngestKind::Binary
136    }
137}
138
139/// Whether a path should be fed to the index. Single front-door replacing the
140/// old `is_code_file` gate.
141#[must_use]
142pub fn is_ingestible(path: &Path) -> bool {
143    classify_path(path).is_ingestible()
144}
145
146/// Bounded heuristic: read the first [`SNIFF_BYTES`] and decide whether the
147/// content is text. A NUL byte or a high ratio of non-text control bytes marks
148/// it binary. Unreadable files are treated as binary (skipped).
149fn looks_textual(path: &Path) -> bool {
150    use std::io::Read;
151
152    let Ok(mut file) = std::fs::File::open(path) else {
153        return false;
154    };
155    let mut buf = [0u8; SNIFF_BYTES];
156    let n = match file.read(&mut buf) {
157        Ok(0) => return true, // empty file: harmless to index
158        Ok(n) => n,
159        Err(_) => return false,
160    };
161    let sample = &buf[..n];
162
163    if sample.contains(&0) {
164        return false;
165    }
166    let suspicious = sample
167        .iter()
168        .filter(|&&b| b < 0x09 || (b > 0x0d && b < 0x20))
169        .count();
170    // Allow a small fraction of control bytes (some text files carry form-feed
171    // etc.) but reject clearly binary content.
172    suspicious * 100 / n.max(1) < 10
173}
174
175#[cfg(test)]
176mod tests {
177    use super::*;
178    use std::io::Write;
179    use std::path::PathBuf;
180
181    fn p(name: &str) -> PathBuf {
182        PathBuf::from(name)
183    }
184
185    #[test]
186    fn code_files_classify_as_code() {
187        assert_eq!(classify_path(&p("src/main.rs")), IngestKind::Code);
188        assert_eq!(classify_path(&p("app/index.ts")), IngestKind::Code);
189        assert!(is_ingestible(&p("src/main.rs")));
190    }
191
192    #[test]
193    fn documents_and_data_are_ingestible() {
194        assert_eq!(classify_path(&p("README.md")), IngestKind::Document);
195        assert_eq!(classify_path(&p("notes.txt")), IngestKind::Document);
196        assert_eq!(classify_path(&p("page.html")), IngestKind::Document);
197        assert_eq!(classify_path(&p("data.csv")), IngestKind::Data);
198        assert_eq!(classify_path(&p("config.yaml")), IngestKind::Data);
199        for f in ["README.md", "data.csv", "config.yaml", "page.html"] {
200            assert!(is_ingestible(&p(f)), "{f} should ingest");
201        }
202    }
203
204    #[test]
205    fn binaries_are_excluded() {
206        for f in ["logo.png", "archive.zip", "lib.so", "app.wasm"] {
207            assert_eq!(classify_path(&p(f)), IngestKind::Binary, "{f}");
208            assert!(!is_ingestible(&p(f)), "{f} must not ingest");
209        }
210    }
211
212    #[test]
213    fn pdf_is_ingestible_via_extractor() {
214        // PDF has a dedicated extractor, so it classifies as an ingestible
215        // document (read through `extractors::extract`, not as UTF-8).
216        assert_eq!(classify_path(&p("report.pdf")), IngestKind::Document);
217        assert_eq!(classify_path(&p("REPORT.PDF")), IngestKind::Document);
218        assert!(is_ingestible(&p("report.pdf")));
219        // Office binaries without an extractor stay excluded.
220        for f in ["paper.docx", "sheet.xlsx", "deck.pptx", "doc.odt"] {
221            assert_eq!(classify_path(&p(f)), IngestKind::Binary, "{f}");
222            assert!(!is_ingestible(&p(f)), "{f} must not ingest");
223        }
224    }
225
226    #[test]
227    fn unknown_extension_is_sniffed() {
228        let dir = tempfile::tempdir().unwrap();
229
230        let textual = dir.path().join("mystery.weirdext");
231        std::fs::write(&textual, "just normal text content\nwith lines\n").unwrap();
232        assert_eq!(classify_path(&textual), IngestKind::Text);
233        assert!(is_ingestible(&textual));
234
235        let binary = dir.path().join("blob.weirdext");
236        let mut f = std::fs::File::create(&binary).unwrap();
237        f.write_all(&[0u8, 1, 2, 3, 0, 255, 254]).unwrap();
238        assert_eq!(classify_path(&binary), IngestKind::Binary);
239        assert!(!is_ingestible(&binary));
240    }
241
242    #[test]
243    fn no_extension_textual_file_ingests() {
244        let dir = tempfile::tempdir().unwrap();
245        let readme = dir.path().join("LICENSE");
246        std::fs::write(&readme, "MIT License\n\nPermission is hereby granted\n").unwrap();
247        assert_eq!(classify_path(&readme), IngestKind::Text);
248    }
249}
lean_ctx/core/ingestion.rs

lean_ctx/core/
ingestion.rs
