lean_ctx/core/
binary_detect.rs

1use std::path::Path;
2
3const BINARY_EXTENSIONS: &[&str] = &[
4    // Data formats
5    "parquet",
6    "avro",
7    "orc",
8    "arrow",
9    "feather",
10    "hdf5",
11    "h5",
12    "npy",
13    "npz",
14    // Databases
15    "db",
16    "sqlite",
17    "sqlite3",
18    "mdb",
19    "accdb",
20    "ldb",
21    // Archives
22    "zip",
23    "gz",
24    "tar",
25    "bz2",
26    "xz",
27    "7z",
28    "rar",
29    "zst",
30    "lz4",
31    "lzma",
32    // Images
33    "png",
34    "jpg",
35    "jpeg",
36    "gif",
37    "webp",
38    "bmp",
39    "ico",
40    "tiff",
41    "tif",
42    "svg",
43    "psd",
44    "raw",
45    "cr2",
46    "nef",
47    "heic",
48    "heif",
49    "avif",
50    // Audio/Video
51    "mp3",
52    "mp4",
53    "wav",
54    "flac",
55    "ogg",
56    "avi",
57    "mkv",
58    "mov",
59    "webm",
60    "m4a",
61    // Executables/Libraries
62    "exe",
63    "dll",
64    "so",
65    "dylib",
66    "o",
67    "a",
68    "obj",
69    "lib",
70    "pdb",
71    "class",
72    "jar",
73    "war",
74    "ear",
75    // Compiled/Bytecode
76    "pyc",
77    "pyo",
78    "whl",
79    "egg",
80    "beam",
81    "wasm",
82    "wast",
83    // ML models
84    "model",
85    "onnx",
86    "pt",
87    "pth",
88    "safetensors",
89    "gguf",
90    "ggml",
91    "tflite",
92    "pb",
93    "h5",
94    "keras",
95    // Serialized
96    "pkl",
97    "pickle",
98    "bin",
99    "dat",
100    "protobuf",
101    // Documents (binary)
102    "pdf",
103    "doc",
104    "docx",
105    "xls",
106    "xlsx",
107    "ppt",
108    "pptx",
109    "odt",
110    "ods",
111    // Fonts
112    "ttf",
113    "otf",
114    "woff",
115    "woff2",
116    "eot",
117    // Disk images
118    "iso",
119    "img",
120    "vmdk",
121    "qcow2",
122];
123
124/// Image formats that LLMs can process visually via multimodal input.
125/// Only formats supported by all major providers (Anthropic, OpenAI, Google).
126const LLM_VIEWABLE_EXTENSIONS: &[&str] = &["png", "jpg", "jpeg", "gif", "webp"];
127
128/// Maximum file size for image passthrough (20 MB).
129pub const IMAGE_MAX_BYTES: u64 = 20 * 1024 * 1024;
130
131/// Fast extension-based binary detection (zero I/O).
132fn has_binary_extension(path: &str) -> bool {
133    Path::new(path)
134        .extension()
135        .and_then(|e| e.to_str())
136        .map(str::to_ascii_lowercase)
137        .is_some_and(|ext| BINARY_EXTENSIONS.contains(&ext.as_str()))
138}
139
140/// Heuristic: read first 8 KB and check for NULL bytes.
141/// Standard method used by `file(1)`, git, etc.
142fn has_binary_content(path: &str) -> bool {
143    let Ok(file) = std::fs::File::open(path) else {
144        return false;
145    };
146    use std::io::Read;
147    let mut buf = [0u8; 8192];
148    let mut reader = std::io::BufReader::new(file);
149    let Ok(n) = reader.read(&mut buf) else {
150        return false;
151    };
152    buf[..n].contains(&0)
153}
154
155/// Returns `true` if the file is likely a binary file.
156/// Checks extension first (zero I/O), falls back to content inspection.
157pub fn is_binary_file(path: &str) -> bool {
158    if has_binary_extension(path) {
159        return true;
160    }
161    has_binary_content(path)
162}
163
164/// Returns `true` if the file is an image format that LLMs can view visually.
165pub fn is_llm_viewable_image(path: &str) -> bool {
166    Path::new(path)
167        .extension()
168        .and_then(|e| e.to_str())
169        .map(str::to_ascii_lowercase)
170        .is_some_and(|ext| LLM_VIEWABLE_EXTENSIONS.contains(&ext.as_str()))
171}
172
173/// Returns the MIME type for an LLM-viewable image, or None if not viewable.
174pub fn image_mime_type(path: &str) -> Option<&'static str> {
175    let ext = Path::new(path)
176        .extension()
177        .and_then(|e| e.to_str())?
178        .to_ascii_lowercase();
179    match ext.as_str() {
180        "png" => Some("image/png"),
181        "jpg" | "jpeg" => Some("image/jpeg"),
182        "gif" => Some("image/gif"),
183        "webp" => Some("image/webp"),
184        _ => None,
185    }
186}
187
188/// Returns a human-readable file type label for common binary extensions.
189fn file_type_label(path: &str) -> &'static str {
190    let ext = Path::new(path)
191        .extension()
192        .and_then(|e| e.to_str())
193        .unwrap_or("");
194    match ext.to_ascii_lowercase().as_str() {
195        "parquet" | "avro" | "orc" | "arrow" | "feather" => "columnar data file",
196        "hdf5" | "h5" | "npy" | "npz" => "scientific data file",
197        "db" | "sqlite" | "sqlite3" => "database file",
198        "zip" | "gz" | "tar" | "bz2" | "xz" | "7z" | "rar" | "zst" => "compressed archive",
199        "png" | "jpg" | "jpeg" | "gif" | "webp" | "bmp" | "ico" | "heic" => "image file",
200        "mp3" | "mp4" | "wav" | "flac" | "ogg" | "avi" | "mkv" | "mov" => "media file",
201        "exe" | "dll" | "so" | "dylib" => "native binary",
202        "wasm" => "WebAssembly binary",
203        "pdf" => "PDF document",
204        "onnx" | "pt" | "pth" | "safetensors" | "gguf" | "ggml" => "ML model file",
205        "pkl" | "pickle" => "serialized object",
206        "pyc" | "pyo" => "Python bytecode",
207        "class" | "jar" | "war" => "Java bytecode",
208        _ => "binary file",
209    }
210}
211
212/// Returns a helpful error message for binary files, including file type and suggestions.
213pub fn binary_file_message(path: &str) -> String {
214    let ext = Path::new(path)
215        .extension()
216        .and_then(|e| e.to_str())
217        .unwrap_or("unknown");
218    let label = file_type_label(path);
219    format!(
220        "Binary file detected (.{ext}, {label}). \
221         lean-ctx cannot read binary files as text. \
222         Use a specialized tool for this file type."
223    )
224}
225
226#[cfg(test)]
227mod tests {
228    use super::*;
229
230    #[test]
231    fn detects_binary_extensions() {
232        assert!(has_binary_extension("data.parquet"));
233        assert!(has_binary_extension("model.onnx"));
234        assert!(has_binary_extension("archive.tar.gz"));
235        assert!(has_binary_extension("photo.PNG"));
236        assert!(has_binary_extension("/path/to/file.sqlite3"));
237    }
238
239    #[test]
240    fn rejects_text_extensions() {
241        assert!(!has_binary_extension("main.rs"));
242        assert!(!has_binary_extension("config.toml"));
243        assert!(!has_binary_extension("README.md"));
244        assert!(!has_binary_extension("script.py"));
245    }
246
247    #[test]
248    fn message_includes_type() {
249        let msg = binary_file_message("data.parquet");
250        assert!(msg.contains("columnar data file"));
251        assert!(msg.contains(".parquet"));
252    }
253
254    #[test]
255    fn message_for_unknown_binary() {
256        let msg = binary_file_message("file.xyz");
257        assert!(msg.contains("binary file"));
258    }
259
260    #[test]
261    fn null_byte_detection() {
262        let dir = std::env::temp_dir().join("lean_ctx_binary_test");
263        std::fs::create_dir_all(&dir).ok();
264
265        let bin_path = dir.join("test.bin");
266        std::fs::write(&bin_path, b"\x00\x01\x02\x03").unwrap();
267        assert!(has_binary_content(bin_path.to_str().unwrap()));
268
269        let txt_path = dir.join("test.txt");
270        std::fs::write(&txt_path, b"hello world").unwrap();
271        assert!(!has_binary_content(txt_path.to_str().unwrap()));
272
273        std::fs::remove_dir_all(&dir).ok();
274    }
275
276    #[test]
277    fn llm_viewable_detects_supported_formats() {
278        assert!(is_llm_viewable_image("photo.png"));
279        assert!(is_llm_viewable_image("photo.PNG"));
280        assert!(is_llm_viewable_image("image.jpg"));
281        assert!(is_llm_viewable_image("image.jpeg"));
282        assert!(is_llm_viewable_image("anim.gif"));
283        assert!(is_llm_viewable_image("modern.webp"));
284        assert!(is_llm_viewable_image("/path/to/file.JPEG"));
285    }
286
287    #[test]
288    fn llm_viewable_rejects_unsupported() {
289        assert!(!is_llm_viewable_image("icon.svg"));
290        assert!(!is_llm_viewable_image("photo.heic"));
291        assert!(!is_llm_viewable_image("image.tiff"));
292        assert!(!is_llm_viewable_image("design.psd"));
293        assert!(!is_llm_viewable_image("photo.avif"));
294        assert!(!is_llm_viewable_image("code.rs"));
295        assert!(!is_llm_viewable_image("data.bin"));
296    }
297
298    #[test]
299    fn mime_type_correct() {
300        assert_eq!(image_mime_type("x.png"), Some("image/png"));
301        assert_eq!(image_mime_type("x.jpg"), Some("image/jpeg"));
302        assert_eq!(image_mime_type("x.jpeg"), Some("image/jpeg"));
303        assert_eq!(image_mime_type("x.gif"), Some("image/gif"));
304        assert_eq!(image_mime_type("x.webp"), Some("image/webp"));
305        assert_eq!(image_mime_type("x.svg"), None);
306        assert_eq!(image_mime_type("x.rs"), None);
307    }
308}
lean_ctx/core/binary_detect.rs

lean_ctx/core/
binary_detect.rs
