lean_ctx/core/
binary_detect.rs

1use std::path::Path;
2
3const BINARY_EXTENSIONS: &[&str] = &[
4    // Data formats
5    "parquet",
6    "avro",
7    "orc",
8    "arrow",
9    "feather",
10    "hdf5",
11    "h5",
12    "npy",
13    "npz",
14    // Databases
15    "db",
16    "sqlite",
17    "sqlite3",
18    "mdb",
19    "accdb",
20    "ldb",
21    // Archives
22    "zip",
23    "gz",
24    "tar",
25    "bz2",
26    "xz",
27    "7z",
28    "rar",
29    "zst",
30    "lz4",
31    "lzma",
32    // Images
33    "png",
34    "jpg",
35    "jpeg",
36    "gif",
37    "webp",
38    "bmp",
39    "ico",
40    "tiff",
41    "tif",
42    // No "svg": it is XML text that agents read and edit like source. A real
43    // binary behind any extension is still caught by the content check.
44    "psd",
45    "raw",
46    "cr2",
47    "nef",
48    "heic",
49    "heif",
50    "avif",
51    // Audio/Video
52    "mp3",
53    "mp4",
54    "wav",
55    "flac",
56    "ogg",
57    "avi",
58    "mkv",
59    "mov",
60    "webm",
61    "m4a",
62    // Executables/Libraries
63    "exe",
64    "dll",
65    "so",
66    "dylib",
67    "o",
68    "a",
69    "obj",
70    "lib",
71    "pdb",
72    "class",
73    "jar",
74    "war",
75    "ear",
76    // Compiled/Bytecode
77    "pyc",
78    "pyo",
79    "whl",
80    "egg",
81    "beam",
82    "wasm",
83    "wast",
84    // ML models
85    "model",
86    "onnx",
87    "pt",
88    "pth",
89    "safetensors",
90    "gguf",
91    "ggml",
92    "tflite",
93    "pb",
94    "h5",
95    "keras",
96    // Serialized
97    "pkl",
98    "pickle",
99    "bin",
100    "dat",
101    "protobuf",
102    // Documents (binary)
103    "pdf",
104    "doc",
105    "docx",
106    "xls",
107    "xlsx",
108    "ppt",
109    "pptx",
110    "odt",
111    "ods",
112    // Fonts
113    "ttf",
114    "otf",
115    "woff",
116    "woff2",
117    "eot",
118    // Disk images
119    "iso",
120    "img",
121    "vmdk",
122    "qcow2",
123];
124
125/// Image formats that LLMs can process visually via multimodal input.
126/// Only formats supported by all major providers (Anthropic, OpenAI, Google).
127const LLM_VIEWABLE_EXTENSIONS: &[&str] = &["png", "jpg", "jpeg", "gif", "webp"];
128
129/// Maximum file size for image passthrough (20 MB).
130pub const IMAGE_MAX_BYTES: u64 = 20 * 1024 * 1024;
131
132/// Fast extension-based binary detection (zero I/O).
133pub fn has_binary_extension(path: &str) -> bool {
134    Path::new(path)
135        .extension()
136        .and_then(|e| e.to_str())
137        .map(str::to_ascii_lowercase)
138        .is_some_and(|ext| BINARY_EXTENSIONS.contains(&ext.as_str()))
139}
140
141/// Heuristic: read first 8 KB and check for NULL bytes.
142/// Standard method used by `file(1)`, git, etc.
143fn has_binary_content(path: &str) -> bool {
144    let Ok(file) = std::fs::File::open(path) else {
145        return false;
146    };
147    use std::io::Read;
148    let mut buf = [0u8; 8192];
149    let mut reader = std::io::BufReader::new(file);
150    let Ok(n) = reader.read(&mut buf) else {
151        return false;
152    };
153    crate::core::text_decode::looks_binary(&buf[..n])
154}
155
156/// Returns `true` if the file is likely a binary file.
157/// Checks extension first (zero I/O), falls back to content inspection.
158pub fn is_binary_file(path: &str) -> bool {
159    if has_binary_extension(path) {
160        return true;
161    }
162    has_binary_content(path)
163}
164
165/// Returns `true` if the file is an image format that LLMs can view visually.
166pub fn is_llm_viewable_image(path: &str) -> bool {
167    Path::new(path)
168        .extension()
169        .and_then(|e| e.to_str())
170        .map(str::to_ascii_lowercase)
171        .is_some_and(|ext| LLM_VIEWABLE_EXTENSIONS.contains(&ext.as_str()))
172}
173
174/// Returns the MIME type for an LLM-viewable image, or None if not viewable.
175pub fn image_mime_type(path: &str) -> Option<&'static str> {
176    let ext = Path::new(path)
177        .extension()
178        .and_then(|e| e.to_str())?
179        .to_ascii_lowercase();
180    match ext.as_str() {
181        "png" => Some("image/png"),
182        "jpg" | "jpeg" => Some("image/jpeg"),
183        "gif" => Some("image/gif"),
184        "webp" => Some("image/webp"),
185        _ => None,
186    }
187}
188
189/// Returns a human-readable file type label for common binary extensions.
190fn file_type_label(path: &str) -> &'static str {
191    let ext = Path::new(path)
192        .extension()
193        .and_then(|e| e.to_str())
194        .unwrap_or("");
195    match ext.to_ascii_lowercase().as_str() {
196        "parquet" | "avro" | "orc" | "arrow" | "feather" => "columnar data file",
197        "hdf5" | "h5" | "npy" | "npz" => "scientific data file",
198        "db" | "sqlite" | "sqlite3" => "database file",
199        "zip" | "gz" | "tar" | "bz2" | "xz" | "7z" | "rar" | "zst" => "compressed archive",
200        "png" | "jpg" | "jpeg" | "gif" | "webp" | "bmp" | "ico" | "heic" => "image file",
201        "mp3" | "mp4" | "wav" | "flac" | "ogg" | "avi" | "mkv" | "mov" => "media file",
202        "exe" | "dll" | "so" | "dylib" => "native binary",
203        "wasm" => "WebAssembly binary",
204        "pdf" => "PDF document",
205        "onnx" | "pt" | "pth" | "safetensors" | "gguf" | "ggml" => "ML model file",
206        "pkl" | "pickle" => "serialized object",
207        "pyc" | "pyo" => "Python bytecode",
208        "class" | "jar" | "war" => "Java bytecode",
209        _ => "binary file",
210    }
211}
212
213/// Returns a helpful error message for binary files, including file type and suggestions.
214pub fn binary_file_message(path: &str) -> String {
215    let ext = Path::new(path)
216        .extension()
217        .and_then(|e| e.to_str())
218        .unwrap_or("unknown");
219    let label = file_type_label(path);
220    if matches!(
221        ext.to_ascii_lowercase().as_str(),
222        "parquet" | "avro" | "orc" | "arrow" | "feather"
223    ) {
224        let size_info = std::fs::metadata(path).ok().map_or_else(
225            || "unknown size".to_string(),
226            |metadata| format_file_size(metadata.len()),
227        );
228        return format!(
229            "Binary {label} (.{ext}, {size_info}). \
230             Cannot read as text. \
231             Inspect with: duckdb \"SELECT * FROM '{path}' LIMIT 5\" \
232             or: python -c \"import pyarrow.parquet as pq; print(pq.read_schema('{path}'))\""
233        );
234    }
235    format!(
236        "Binary file detected (.{ext}, {label}). \
237         lean-ctx cannot read binary files as text. \
238         Use a specialized tool for this file type."
239    )
240}
241
242fn format_file_size(bytes: u64) -> String {
243    if bytes >= 1_073_741_824 {
244        format!("{:.1} GB", bytes as f64 / 1_073_741_824.0)
245    } else if bytes >= 1_048_576 {
246        format!("{:.1} MB", bytes as f64 / 1_048_576.0)
247    } else if bytes >= 1024 {
248        format!("{:.1} KB", bytes as f64 / 1024.0)
249    } else {
250        format!("{bytes} B")
251    }
252}
253
254#[cfg(test)]
255mod tests {
256    use super::{
257        binary_file_message, has_binary_content, has_binary_extension, image_mime_type,
258        is_llm_viewable_image,
259    };
260
261    #[test]
262    fn detects_binary_extensions() {
263        assert!(has_binary_extension("data.parquet"));
264        assert!(has_binary_extension("model.onnx"));
265        assert!(has_binary_extension("archive.tar.gz"));
266        assert!(has_binary_extension("photo.PNG"));
267        assert!(has_binary_extension("/path/to/file.sqlite3"));
268    }
269
270    #[test]
271    fn rejects_text_extensions() {
272        assert!(!has_binary_extension("main.rs"));
273        assert!(!has_binary_extension("config.toml"));
274        assert!(!has_binary_extension("README.md"));
275        assert!(!has_binary_extension("script.py"));
276        assert!(!has_binary_extension("icons/logo.svg"));
277        assert!(!has_binary_extension("ICON.SVG"));
278    }
279
280    #[test]
281    fn message_includes_type() {
282        let msg = binary_file_message("data.parquet");
283        assert!(msg.contains("columnar data file"));
284        assert!(msg.contains(".parquet"));
285    }
286
287    #[test]
288    fn parquet_message_includes_tool_suggestion() {
289        let msg = binary_file_message("data.parquet");
290        assert!(msg.contains("duckdb"));
291        assert!(msg.contains("columnar data"));
292    }
293
294    #[test]
295    fn message_for_unknown_binary() {
296        let msg = binary_file_message("file.xyz");
297        assert!(msg.contains("binary file"));
298    }
299
300    #[test]
301    fn null_byte_detection() {
302        let dir = std::env::temp_dir().join("lean_ctx_binary_test");
303        std::fs::create_dir_all(&dir).ok();
304
305        let bin_path = dir.join("test.bin");
306        std::fs::write(&bin_path, b"\x00\x01\x02\x03").unwrap();
307        assert!(has_binary_content(bin_path.to_str().unwrap()));
308
309        let txt_path = dir.join("test.txt");
310        std::fs::write(&txt_path, b"hello world").unwrap();
311        assert!(!has_binary_content(txt_path.to_str().unwrap()));
312
313        std::fs::remove_dir_all(&dir).ok();
314    }
315
316    #[test]
317    fn llm_viewable_detects_supported_formats() {
318        assert!(is_llm_viewable_image("photo.png"));
319        assert!(is_llm_viewable_image("photo.PNG"));
320        assert!(is_llm_viewable_image("image.jpg"));
321        assert!(is_llm_viewable_image("image.jpeg"));
322        assert!(is_llm_viewable_image("anim.gif"));
323        assert!(is_llm_viewable_image("modern.webp"));
324        assert!(is_llm_viewable_image("/path/to/file.JPEG"));
325    }
326
327    #[test]
328    fn llm_viewable_rejects_unsupported() {
329        assert!(!is_llm_viewable_image("icon.svg"));
330        assert!(!is_llm_viewable_image("photo.heic"));
331        assert!(!is_llm_viewable_image("image.tiff"));
332        assert!(!is_llm_viewable_image("design.psd"));
333        assert!(!is_llm_viewable_image("photo.avif"));
334        assert!(!is_llm_viewable_image("code.rs"));
335        assert!(!is_llm_viewable_image("data.bin"));
336    }
337
338    #[test]
339    fn mime_type_correct() {
340        assert_eq!(image_mime_type("x.png"), Some("image/png"));
341        assert_eq!(image_mime_type("x.jpg"), Some("image/jpeg"));
342        assert_eq!(image_mime_type("x.jpeg"), Some("image/jpeg"));
343        assert_eq!(image_mime_type("x.gif"), Some("image/gif"));
344        assert_eq!(image_mime_type("x.webp"), Some("image/webp"));
345        assert_eq!(image_mime_type("x.svg"), None);
346        assert_eq!(image_mime_type("x.rs"), None);
347    }
348}
lean_ctx/core/binary_detect.rs

lean_ctx/core/
binary_detect.rs
