scribe_scanner/

filtering.rs

//! High-performance file filtering with early content reads and strict pre-filtering.
//!
//! This module implements the performance-critical pre-filtering logic that dramatically
//! reduces work by eliminating files before expensive operations like content analysis,
//! git lookups, and heuristic computation.

use fxhash::FxHashSet;
use memchr::memmem;
use once_cell::sync::Lazy;
use std::collections::HashSet;
use std::path::{Path, PathBuf};

/// Cold file extensions that should be filtered out early
static COLD_EXTENSIONS: Lazy<FxHashSet<&'static str>> = Lazy::new(|| {
    [
        // Documentation that's rarely code-relevant
        "md", "txt", "rst", "adoc", "wiki", // Media files
        "png", "jpg", "jpeg", "gif", "bmp", "ico", "svg", "webp", "tiff", "mp3", "mp4", "avi",
        "mkv", "mov", "wmv", "flv", "webm", "m4v", "wav", "flac", "ogg", "aac", "wma",
        // Archives and packages
        "zip", "tar", "gz", "bz2", "xz", "7z", "rar", "jar", "war", "ear",
        // Binary executables
        "exe", "dll", "so", "dylib", "a", "lib", "bin", "out", // Office documents
        "pdf", "doc", "docx", "xls", "xlsx", "ppt", "pptx", "odt", "ods", "odp",
        // Fonts
        "ttf", "otf", "woff", "woff2", "eot", // Cache/temp files
        "tmp", "temp", "cache", "log", "bak", "swp", "swo", // Generated/minified
        "min.js", "min.css",
    ]
    .into_iter()
    .collect()
});

/// Hot file extensions that are likely to contain important code
static HOT_EXTENSIONS: Lazy<FxHashSet<&'static str>> = Lazy::new(|| {
    [
        // Core programming languages
        "rs",
        "py",
        "js",
        "ts",
        "jsx",
        "tsx",
        "go",
        "java",
        "c",
        "cpp",
        "h",
        "hpp",
        "cs",
        "php",
        "rb",
        "swift",
        "kt",
        "scala",
        "clj",
        "hs",
        "elm",
        "ml",
        "ocaml",
        // Configuration and markup with logic
        "json",
        "yaml",
        "yml",
        "toml",
        "xml",
        "html",
        "css",
        "scss",
        "less",
        "sass",
        // Scripts and configs
        "sh",
        "bash",
        "zsh",
        "fish",
        "ps1",
        "cmd",
        "bat",
        "dockerfile",
        "makefile",
        // Database and query languages
        "sql",
        "graphql",
        "prisma",
    ]
    .into_iter()
    .collect()
});

/// Vendor/generated directory patterns to skip entirely
static COLD_DIRS: Lazy<FxHashSet<&'static str>> = Lazy::new(|| {
    [
        "node_modules",
        "__pycache__",
        ".pytest_cache",
        ".mypy_cache",
        "target",
        "build",
        "dist",
        ".git",
        ".hg",
        ".svn",
        "vendor",
        "third_party",
        "external",
        "deps",
        ".idea",
        ".vscode",
        ".vs",
        ".gradle",
        ".maven",
        "coverage",
        ".coverage",
        ".nyc_output",
        "logs",
        "tmp",
        "temp",
        ".tmp",
        ".temp",
    ]
    .into_iter()
    .collect()
});

/// Binary content detection patterns (first 512 bytes)
static BINARY_MARKERS: Lazy<Vec<&'static [u8]>> = Lazy::new(|| {
    vec![
        b"\x7fELF",          // ELF binaries
        b"MZ",               // Windows PE
        b"\xca\xfe\xba\xbe", // Java class files
        b"\xfe\xed\xfa\xce", // Mach-O binaries
        b"\x89PNG",          // PNG images
        b"\xff\xd8\xff",     // JPEG images
        b"GIF8",             // GIF images
        b"RIFF",             // WAV/AVI files
        b"%PDF",             // PDF files
        b"PK\x03\x04",       // ZIP files
    ]
});

/// Maximum file size for content-based analysis (8MB)
const MAX_CONTENT_SIZE: u64 = 8 * 1024 * 1024;

/// Size for binary detection sample (512 bytes)
const BINARY_SAMPLE_SIZE: usize = 512;

/// High-performance file filter with strict pre-filtering
#[derive(Debug)]
pub struct FileFilter {
    /// Custom extension allowlist (if set, only these are allowed)
    allow_extensions: Option<FxHashSet<String>>,
    /// Custom extension denylist (these are always blocked)  
    deny_extensions: FxHashSet<String>,
    /// Maximum file size to process
    max_file_size: u64,
    /// Whether to include hidden files
    include_hidden: bool,
    /// Whether to perform binary content detection
    binary_detection: bool,
    /// Performance counters
    stats: FilterStats,
}

/// Performance statistics for filtering operations
#[derive(Debug, Default, Clone)]
pub struct FilterStats {
    pub files_walked: u64,
    pub dirs_skipped: u64,
    pub extension_filtered: u64,
    pub size_filtered: u64,
    pub binary_filtered: u64,
    pub passed_filter: u64,
    pub bytes_read_for_detection: u64,
}

/// Result of pre-filtering a single file
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum FilterResult {
    /// File should be processed
    Include,
    /// File should be skipped with reason
    Exclude(FilterReason),
}

/// Reasons for filtering out files
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum FilterReason {
    ColdExtension,
    ColdDirectory,
    TooLarge(u64),
    Hidden,
    Binary,
    CustomExtensionFilter,
}

impl FileFilter {
    /// Create a new file filter with performance-optimized defaults
    pub fn new() -> Self {
        Self {
            allow_extensions: None,
            deny_extensions: FxHashSet::default(),
            max_file_size: MAX_CONTENT_SIZE,
            include_hidden: false,
            binary_detection: true,
            stats: FilterStats::default(),
        }
    }

    /// Set custom extension allowlist (only these extensions will be processed)
    pub fn with_allow_extensions(mut self, extensions: Vec<String>) -> Self {
        self.allow_extensions = Some(extensions.into_iter().map(|e| e.to_lowercase()).collect());
        self
    }

    /// Add extensions to the deny list
    pub fn with_deny_extensions(mut self, extensions: Vec<String>) -> Self {
        self.deny_extensions = extensions.into_iter().map(|e| e.to_lowercase()).collect();
        self
    }

    /// Set maximum file size
    pub fn with_max_file_size(mut self, size: u64) -> Self {
        self.max_file_size = size;
        self
    }

    /// Set whether to include hidden files
    pub fn with_include_hidden(mut self, include: bool) -> Self {
        self.include_hidden = include;
        self
    }

    /// Set whether to perform binary detection
    pub fn with_binary_detection(mut self, detect: bool) -> Self {
        self.binary_detection = detect;
        self
    }

    /// Pre-filter a file path without reading contents
    pub fn pre_filter_path(&mut self, path: &Path) -> FilterResult {
        self.stats.files_walked += 1;

        // Check hidden files
        if !self.include_hidden {
            if let Some(name) = path.file_name() {
                if name.to_string_lossy().starts_with('.') {
                    return FilterResult::Exclude(FilterReason::Hidden);
                }
            }
        }

        // Check for cold directories in path
        for component in path.components() {
            if let std::path::Component::Normal(name) = component {
                if COLD_DIRS.contains(name.to_str().unwrap_or("")) {
                    self.stats.dirs_skipped += 1;
                    return FilterResult::Exclude(FilterReason::ColdDirectory);
                }
            }
        }

        // Get file extension
        let extension = path
            .extension()
            .and_then(|ext| ext.to_str())
            .unwrap_or("")
            .to_lowercase();

        // Apply custom extension filters
        if let Some(ref allow_list) = self.allow_extensions {
            if !allow_list.contains(&extension) {
                self.stats.extension_filtered += 1;
                return FilterResult::Exclude(FilterReason::CustomExtensionFilter);
            }
        }

        if self.deny_extensions.contains(&extension) {
            self.stats.extension_filtered += 1;
            return FilterResult::Exclude(FilterReason::CustomExtensionFilter);
        }

        // Check against cold extensions
        if COLD_EXTENSIONS.contains(extension.as_str()) {
            self.stats.extension_filtered += 1;
            return FilterResult::Exclude(FilterReason::ColdExtension);
        }

        FilterResult::Include
    }

    /// Full filter including file size and binary detection
    pub async fn filter_file(&mut self, path: &Path) -> FilterResult {
        // First apply path-based filtering
        match self.pre_filter_path(path) {
            FilterResult::Exclude(reason) => return FilterResult::Exclude(reason),
            FilterResult::Include => {}
        }

        // Check file size
        if let Ok(metadata) = tokio::fs::metadata(path).await {
            if metadata.len() > self.max_file_size {
                self.stats.size_filtered += 1;
                return FilterResult::Exclude(FilterReason::TooLarge(metadata.len()));
            }

            // Binary detection if enabled
            if self.binary_detection && self.should_check_binary(path) {
                if self.is_binary_file(path).await {
                    self.stats.binary_filtered += 1;
                    return FilterResult::Exclude(FilterReason::Binary);
                }
            }
        }

        self.stats.passed_filter += 1;
        FilterResult::Include
    }

    /// Check if we should perform binary detection for this file
    fn should_check_binary(&self, path: &Path) -> bool {
        let extension = path
            .extension()
            .and_then(|ext| ext.to_str())
            .unwrap_or("")
            .to_lowercase();

        // Skip binary detection for known text extensions
        if HOT_EXTENSIONS.contains(extension.as_str()) {
            return false;
        }

        // Skip for files with no extension (often text)
        if extension.is_empty() {
            return false;
        }

        true
    }

    /// Fast binary file detection using content sampling
    pub async fn is_binary_file(&mut self, path: &Path) -> bool {
        match tokio::fs::File::open(path).await {
            Ok(mut file) => {
                use tokio::io::AsyncReadExt;

                let mut buffer = vec![0u8; BINARY_SAMPLE_SIZE];
                match file.read(&mut buffer).await {
                    Ok(bytes_read) => {
                        self.stats.bytes_read_for_detection += bytes_read as u64;
                        buffer.truncate(bytes_read);

                        self.detect_binary_content(&buffer)
                    }
                    Err(_) => false, // Assume text if we can't read
                }
            }
            Err(_) => false, // Assume text if we can't open
        }
    }

    /// Detect binary content using multiple heuristics
    fn detect_binary_content(&self, content: &[u8]) -> bool {
        // Check for known binary markers
        for marker in BINARY_MARKERS.iter() {
            if content.starts_with(marker) {
                return true;
            }
        }

        // Null byte check (classic binary detection)
        if memchr::memchr(0, content).is_some() {
            return true;
        }

        // High percentage of non-printable bytes
        let non_printable = content
            .iter()
            .filter(|&&b| b < 32 && b != b'\t' && b != b'\n' && b != b'\r')
            .count();

        let ratio = non_printable as f64 / content.len() as f64;
        ratio > 0.05 // More than 5% non-printable
    }

    /// Get filtering statistics
    pub fn stats(&self) -> &FilterStats {
        &self.stats
    }

    /// Reset statistics
    pub fn reset_stats(&mut self) {
        self.stats = FilterStats::default();
    }
}

impl Default for FileFilter {
    fn default() -> Self {
        Self::new()
    }
}

/// Directory-level filtering for efficient tree traversal
#[derive(Debug)]
pub struct DirectoryFilter {
    cold_dirs: FxHashSet<String>,
    stats: DirectoryFilterStats,
}

#[derive(Debug, Default)]
pub struct DirectoryFilterStats {
    pub dirs_walked: u64,
    pub dirs_skipped: u64,
}

impl DirectoryFilter {
    pub fn new() -> Self {
        Self {
            cold_dirs: COLD_DIRS.iter().map(|s| s.to_string()).collect(),
            stats: DirectoryFilterStats::default(),
        }
    }

    pub fn with_additional_cold_dirs(mut self, dirs: Vec<String>) -> Self {
        self.cold_dirs.extend(dirs);
        self
    }

    /// Check if a directory should be skipped entirely
    pub fn should_skip_directory(&mut self, path: &Path) -> bool {
        self.stats.dirs_walked += 1;

        if let Some(name) = path.file_name() {
            if let Some(name_str) = name.to_str() {
                if self.cold_dirs.contains(name_str) {
                    self.stats.dirs_skipped += 1;
                    return true;
                }
            }
        }

        false
    }

    pub fn stats(&self) -> &DirectoryFilterStats {
        &self.stats
    }
}

impl Default for DirectoryFilter {
    fn default() -> Self {
        Self::new()
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use tempfile::TempDir;
    use tokio::fs;

    #[tokio::test]
    async fn test_cold_extension_filtering() {
        let mut filter = FileFilter::new();

        assert_eq!(
            filter.pre_filter_path(Path::new("test.png")),
            FilterResult::Exclude(FilterReason::ColdExtension)
        );

        assert_eq!(
            filter.pre_filter_path(Path::new("code.rs")),
            FilterResult::Include
        );
    }

    #[tokio::test]
    async fn test_cold_directory_filtering() {
        let mut filter = FileFilter::new();

        assert_eq!(
            filter.pre_filter_path(Path::new("node_modules/package/index.js")),
            FilterResult::Exclude(FilterReason::ColdDirectory)
        );

        assert_eq!(
            filter.pre_filter_path(Path::new("src/main.rs")),
            FilterResult::Include
        );
    }

    #[tokio::test]
    async fn test_custom_extension_filtering() {
        let mut filter =
            FileFilter::new().with_allow_extensions(vec!["rs".to_string(), "py".to_string()]);

        assert_eq!(
            filter.pre_filter_path(Path::new("test.js")),
            FilterResult::Exclude(FilterReason::CustomExtensionFilter)
        );

        assert_eq!(
            filter.pre_filter_path(Path::new("test.rs")),
            FilterResult::Include
        );
    }

    #[tokio::test]
    async fn test_file_size_filtering() {
        // Create test file in current directory to avoid tmp path issues
        // Use .rs extension which is in HOT_EXTENSIONS, not COLD_EXTENSIONS
        let large_file = Path::new("test_large_file.rs");

        // Create a file larger than 1KB
        let content = "x".repeat(2000);
        fs::write(&large_file, &content).await.unwrap();

        let mut filter = FileFilter::new().with_max_file_size(1000);

        let result = filter.filter_file(&large_file).await;

        // Clean up test file
        let _ = fs::remove_file(&large_file).await;

        match result {
            FilterResult::Exclude(FilterReason::TooLarge(_)) => {}
            other => panic!("Expected TooLarge, got {:?}", other),
        }
    }

    #[tokio::test]
    async fn test_binary_detection() {
        let temp_dir = TempDir::new().unwrap();

        // Create a subdirectory that won't match COLD_DIRS
        let test_dir = temp_dir.path().join("project");
        fs::create_dir_all(&test_dir).await.unwrap();

        // Create a binary file with null bytes
        let binary_file = test_dir.join("binary.dat");
        fs::write(&binary_file, &[0u8, 1u8, 2u8, 0u8])
            .await
            .unwrap();

        // Create a text file
        let text_file = test_dir.join("text.txt");
        fs::write(&text_file, "Hello, world!").await.unwrap();

        let mut filter = FileFilter::new();

        // Test that binary files are detected correctly
        // Since the temp dir path contains "tmp", we need to test binary detection
        // on files that don't get filtered by cold directory first
        assert!(filter.is_binary_file(&binary_file).await);
        assert!(!filter.is_binary_file(&text_file).await);
    }

    #[tokio::test]
    async fn test_hidden_file_filtering() {
        let mut filter = FileFilter::new().with_include_hidden(false);

        assert_eq!(
            filter.pre_filter_path(Path::new(".hidden")),
            FilterResult::Exclude(FilterReason::Hidden)
        );

        let mut filter = FileFilter::new().with_include_hidden(true);

        assert_eq!(
            filter.pre_filter_path(Path::new(".hidden")),
            FilterResult::Include
        );
    }

    #[test]
    fn test_binary_content_detection() {
        let filter = FileFilter::new();

        // ELF binary
        assert!(filter.detect_binary_content(b"\x7fELF\x01\x01\x01"));

        // PDF file
        assert!(filter.detect_binary_content(b"%PDF-1.4\n"));

        // File with null bytes
        assert!(filter.detect_binary_content(b"text\x00more text"));

        // Regular text
        assert!(!filter.detect_binary_content(b"Hello, world!\n"));

        // Text with tabs and newlines
        assert!(!filter.detect_binary_content(b"fn main() {\n\tprintln!(\"Hello\");\n}"));
    }

    #[test]
    fn test_directory_filtering() {
        let mut dir_filter = DirectoryFilter::new();

        assert!(dir_filter.should_skip_directory(Path::new("node_modules")));
        assert!(dir_filter.should_skip_directory(Path::new("target")));
        assert!(!dir_filter.should_skip_directory(Path::new("src")));

        assert_eq!(dir_filter.stats().dirs_walked, 3);
        assert_eq!(dir_filter.stats().dirs_skipped, 2);
    }

    #[test]
    fn test_filter_statistics() {
        let mut filter = FileFilter::new();

        // Test various filtering scenarios
        filter.pre_filter_path(Path::new("test.rs")); // Include
        filter.pre_filter_path(Path::new("test.png")); // Cold extension
        filter.pre_filter_path(Path::new("node_modules/pkg/index.js")); // Cold dir
        filter.pre_filter_path(Path::new(".hidden")); // Hidden

        let stats = filter.stats();
        assert_eq!(stats.files_walked, 4);
        assert_eq!(stats.extension_filtered, 1);
        assert_eq!(stats.dirs_skipped, 1);
        assert_eq!(stats.passed_filter, 0); // pre_filter_path doesn't update passed_filter
    }
}