sherlock_io/
lib.rs

1pub mod detector;
2pub mod languages;
3pub mod patterns;
4pub mod reporter;
5pub mod scanner;
6
7pub use detector::{Analysis, LanguageDetector, LanguageStats};
8pub use languages::{LanguageCategory, LanguageInfo, LanguageRegistry};
9pub use reporter::Reporter;
10pub use scanner::{FileInfo, Scanner};
11
12// Re-export the OutputFormat for convenience
13pub use crate::cli::OutputFormat;
14
15mod cli;
16
17#[cfg(test)]
18mod tests {
19    use super::*;
20    use tempfile::TempDir;
21    use std::fs;
22
23    /// Test data for language detection
24    pub struct TestCase {
25        pub language: String,
26        pub extension: String,
27        pub content: String,
28        pub filename: String,
29    }
30
31    /// Generate minimal sample content that matches language patterns
32    fn generate_sample_content(language: &str) -> String {
33        match language {
34            "Rust" => "fn main() {\n    println!(\"Hello, world!\");\n}".to_string(),
35            "Python" => "def hello():\n    print(\"Hello, world!\")\n\nif __name__ == \"__main__\":\n    hello()".to_string(),
36            "JavaScript" => "function hello() {\n    console.log(\"Hello, world!\");\n}\n\nhello();".to_string(),
37            "TypeScript" => "interface Greeting {\n    message: string;\n}\n\nfunction hello(): void {\n    console.log(\"Hello, world!\");\n}".to_string(),
38            "Go" => "package main\n\nimport \"fmt\"\n\nfunc main() {\n    fmt.Println(\"Hello, world!\")\n}".to_string(),
39            "Java" => "public class Hello {\n    public static void main(String[] args) {\n        System.out.println(\"Hello, world!\");\n    }\n}".to_string(),
40            "C" => "#include <stdio.h>\n\nint main() {\n    printf(\"Hello, world!\\n\");\n    return 0;\n}".to_string(),
41            "C++" => "#include <iostream>\n\nint main() {\n    std::cout << \"Hello, world!\" << std::endl;\n    return 0;\n}".to_string(),
42            "C#" => "using System;\n\nnamespace HelloWorld {\n    public class Program {\n        public static void Main() {\n            Console.WriteLine(\"Hello, world!\");\n        }\n    }\n}".to_string(),
43            "PHP" => "<?php\nfunction hello() {\n    echo \"Hello, world!\";\n}\nhello();\n?>".to_string(),
44            "Ruby" => "def hello\n    puts \"Hello, world!\"\nend\n\nhello".to_string(),
45            "Swift" => "import Foundation\n\nfunc hello() {\n    print(\"Hello, world!\")\n}\n\nhello()".to_string(),
46            "Kotlin" => "fun main() {\n    println(\"Hello, world!\")\n}".to_string(),
47            "HTML" => "<!DOCTYPE html>\n<html>\n<head>\n    <title>Hello</title>\n</head>\n<body>\n    <h1>Hello, world!</h1>\n</body>\n</html>".to_string(),
48            "CSS" => "body {\n    margin: 0;\n    padding: 0;\n    font-family: Arial, sans-serif;\n}".to_string(),
49            "JSON" => "{\n    \"message\": \"Hello, world!\",\n    \"version\": \"1.0.0\",\n    \"active\": true\n}".to_string(),
50            "YAML" => "---\nmessage: \"Hello, world!\"\nversion: 1.0.0\nactive: true".to_string(),
51            "XML" => "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<root>\n    <message>Hello, world!</message>\n</root>".to_string(),
52            "Markdown" => "# Hello World\n\nThis is a **markdown** document.\n\n```code\nSample code block\n```".to_string(),
53            "Shell" => "#!/bin/bash\necho \"Hello, world!\"\nif [ $? -eq 0 ]; then\n    echo \"Success\"\nfi".to_string(),
54            _ => format!("// Sample {} code\nfunction hello() {{\n    return \"Hello, world!\";\n}}", language),
55        }
56    }
57
58    #[test]
59    fn test_language_detection_by_extension() {
60        let detector = LanguageDetector::new().expect("Failed to create detector");
61        
62        // Test major programming languages
63        let test_cases = vec![
64            ("Rust", "rs"),
65            ("Python", "py"),
66            ("JavaScript", "js"),
67            ("TypeScript", "ts"),
68            ("Go", "go"),
69            ("Java", "java"),
70            ("C", "c"),
71            ("C++", "cpp"),
72            ("C#", "cs"),
73            ("PHP", "php"),
74            ("Ruby", "rb"),
75            ("Swift", "swift"),
76            ("Kotlin", "kt"),
77            ("HTML", "html"),
78            ("CSS", "css"),
79            ("JSON", "json"),
80            ("YAML", "yml"),
81            ("XML", "xml"), // Should now work correctly with improved detection
82            ("Markdown", "md"),
83            ("Shell", "sh"),
84        ];
85        
86        for (language, extension) in test_cases {
87            let temp_dir = TempDir::new().expect("Failed to create temp dir");
88            let filename = format!("test.{}", extension);
89            let file_path = temp_dir.path().join(&filename);
90            
91            let content = generate_sample_content(language);
92            fs::write(&file_path, &content)
93                .expect("Failed to write test file");
94            
95            let file_info = FileInfo {
96                path: file_path.clone(),
97                size: content.len() as u64,
98                extension: Some(extension.to_string()),
99                is_binary: false,
100            };
101            
102            let files = vec![file_info];
103            let analysis = detector.analyze_files(&files)
104                .expect("Failed to analyze files");
105            
106            // Check that the language was detected
107            assert!(!analysis.stats.is_empty(), 
108                "No language detected for {} file: {}", 
109                language, filename);
110            
111            // Check that the detected language matches expected
112            let detected_lang = &analysis.stats[0].language;
113            assert_eq!(detected_lang, language,
114                "Expected {} but detected {} for file: {}", 
115                language, detected_lang, filename);
116        }
117    }
118
119    #[test]
120    fn test_language_detection_by_content() {
121        let detector = LanguageDetector::new().expect("Failed to create detector");
122        
123        // Test content-based detection for files without extensions
124        // Focus on highly distinctive patterns that are less ambiguous
125        let content_tests = vec![
126            ("#!/usr/bin/env python3\nprint('hello')", "Python"),
127            ("#!/bin/bash\necho hello", "Shell"),
128            ("#!/usr/bin/env node\nconsole.log('hello')", "JavaScript"),
129            ("<?php\necho 'Hello World';\n?>", "PHP"),
130            ("#include <stdio.h>\nint main() { return 0; }", "C"),
131        ];
132        
133        let mut successful_detections = 0;
134        let total_tests = content_tests.len();
135        
136        for (content, expected_lang) in content_tests {
137            let temp_dir = TempDir::new().expect("Failed to create temp dir");
138            let file_path = temp_dir.path().join("testfile"); // No extension
139            
140            fs::write(&file_path, content)
141                .expect("Failed to write test file");
142            
143            let file_info = FileInfo {
144                path: file_path.clone(),
145                size: content.len() as u64,
146                extension: None, // No extension to force content-based detection
147                is_binary: false,
148            };
149            
150            let files = vec![file_info];
151            let analysis = detector.analyze_files(&files)
152                .expect("Failed to analyze files");
153            
154            if !analysis.stats.is_empty() {
155                let detected_lang = &analysis.stats[0].language;
156                if detected_lang == expected_lang {
157                    successful_detections += 1;
158                } else {
159                    println!("Content detection mismatch: expected {} but got {} for: {}", 
160                        expected_lang, detected_lang, content.lines().next().unwrap_or(content));
161                }
162            } else {
163                println!("No language detected for content: {}", content.lines().next().unwrap_or(content));
164            }
165        }
166        
167        // Content-based detection should work for highly distinctive patterns
168        // We expect at least 80% accuracy for shebang and very specific patterns
169        let success_rate = (successful_detections as f64 / total_tests as f64) * 100.0;
170        assert!(success_rate >= 80.0, 
171            "Content-based detection success rate too low: {:.1}% ({}/{})", 
172            success_rate, successful_detections, total_tests);
173    }
174
175    #[test]
176    fn test_special_files_detection() {
177        let detector = LanguageDetector::new().expect("Failed to create detector");
178        
179        let special_files = vec![
180            ("Dockerfile", "FROM ubuntu:20.04\nRUN apt-get update", "Dockerfile"),
181            ("Makefile", "all:\n\techo 'Building...'\n\n.PHONY: all", "Makefile"),
182            ("Rakefile", "task :default do\n  puts 'Hello'\nend", "Ruby"),
183            ("Gemfile", "source 'https://rubygems.org'\ngem 'rails'", "Ruby"),
184        ];
185        
186        for (filename, content, expected_lang) in special_files {
187            let temp_dir = TempDir::new().expect("Failed to create temp dir");
188            let file_path = temp_dir.path().join(filename);
189            
190            fs::write(&file_path, content)
191                .expect("Failed to write test file");
192            
193            let file_info = FileInfo {
194                path: file_path.clone(),
195                size: content.len() as u64,
196                extension: None,
197                is_binary: false,
198            };
199            
200            let files = vec![file_info];
201            let analysis = detector.analyze_files(&files)
202                .expect("Failed to analyze files");
203            
204            assert!(!analysis.stats.is_empty(), 
205                "No language detected for special file: {}", filename);
206            
207            let detected_lang = &analysis.stats[0].language;
208            assert_eq!(detected_lang, expected_lang,
209                "Expected {} but detected {} for special file: {}", 
210                expected_lang, detected_lang, filename);
211        }
212    }
213
214    #[test]
215    fn test_multiple_extensions_same_language() {
216        let detector = LanguageDetector::new().expect("Failed to create detector");
217        
218        // Test that different extensions for the same language are detected correctly
219        let multi_ext_tests = vec![
220            ("Python", vec!["py", "pyw", "py3", "pyi"]),
221            ("JavaScript", vec!["js", "mjs", "cjs", "es6"]),
222            ("TypeScript", vec!["ts", "mts", "cts"]),
223            ("C++", vec!["cpp", "cc", "cxx", "hpp"]),
224        ];
225        
226        for (language, extensions) in multi_ext_tests {
227            for ext in extensions {
228                let temp_dir = TempDir::new().expect("Failed to create temp dir");
229                let filename = format!("test.{}", ext);
230                let file_path = temp_dir.path().join(&filename);
231                
232                let content = generate_sample_content(language);
233                fs::write(&file_path, &content)
234                    .expect("Failed to write test file");
235                
236                let file_info = FileInfo {
237                    path: file_path.clone(),
238                    size: content.len() as u64,
239                    extension: Some(ext.to_string()),
240                    is_binary: false,
241                };
242                
243                let files = vec![file_info];
244                let analysis = detector.analyze_files(&files)
245                    .expect("Failed to analyze files");
246                
247                assert!(!analysis.stats.is_empty(), 
248                    "No language detected for {}.{}", language, ext);
249                
250                let detected_lang = &analysis.stats[0].language;
251                assert_eq!(detected_lang, language,
252                    "Expected {} but detected {} for extension: {}", 
253                    language, detected_lang, ext);
254            }
255        }
256    }
257
258    #[test]
259    fn test_analysis_statistics() {
260        let detector = LanguageDetector::new().expect("Failed to create detector");
261        let temp_dir = TempDir::new().expect("Failed to create temp dir");
262        
263        // Create multiple test files
264        let test_files = vec![
265            ("test1.rs", "fn main() { println!(\"Hello\"); }", "Rust"),
266            ("test2.py", "def hello():\n    print('Hello')", "Python"),
267            ("test3.js", "function hello() { console.log('Hello'); }", "JavaScript"),
268            ("test4.rs", "use std::collections::HashMap;", "Rust"), // Another Rust file
269        ];
270        
271        let mut file_infos = Vec::new();
272        
273        for (filename, content, _) in &test_files {
274            let file_path = temp_dir.path().join(filename);
275            fs::write(&file_path, content).expect("Failed to write test file");
276            
277            let extension = filename.split('.').last().map(|s| s.to_string());
278            file_infos.push(FileInfo {
279                path: file_path,
280                size: content.len() as u64,
281                extension,
282                is_binary: false,
283            });
284        }
285        
286        let analysis = detector.analyze_files(&file_infos)
287            .expect("Failed to analyze files");
288        
289        // Check analysis statistics
290        assert_eq!(analysis.total_files, 4);
291        assert!(analysis.total_bytes > 0);
292        
293        // Should have 3 different languages (Rust, Python, JavaScript)
294        assert_eq!(analysis.stats.len(), 3);
295        
296        // Rust should have 2 files (highest count)
297        let rust_stats = analysis.stats.iter()
298            .find(|s| s.language == "Rust")
299            .expect("Rust should be detected");
300        assert_eq!(rust_stats.file_count, 2);
301        
302        // Check percentages add up to 100%
303        let total_percentage: f64 = analysis.stats.iter()
304            .map(|s| s.percentage)
305            .sum();
306        assert!((total_percentage - 100.0).abs() < 0.1, 
307            "Percentages should add up to 100%");
308    }
309}
sherlock_io/lib.rs

sherlock_io/
lib.rs
