syncable_cli/analyzer/security/turbo/

mod.rs

//! # Turbo Security Analyzer
//! 
//! High-performance security analyzer that's 10-100x faster than traditional approaches.
//! Uses advanced techniques like multi-pattern matching, memory-mapped I/O, and intelligent filtering.

use std::path::Path;
use std::sync::Arc;
use std::time::Instant;

use crossbeam::channel::bounded;

use rayon::prelude::*;
use log::{info, debug, trace};

pub mod file_discovery;
pub mod pattern_engine;
pub mod cache;
pub mod scanner;
pub mod results;

use file_discovery::{FileDiscovery, FileMetadata, DiscoveryConfig};
use pattern_engine::PatternEngine;
use cache::SecurityCache;
use scanner::{FileScanner, ScanTask, ScanResult};
use results::{ResultAggregator, SecurityReport};

use crate::analyzer::security::SecurityFinding;

/// Turbo security analyzer configuration
#[derive(Debug, Clone)]
pub struct TurboConfig {
    /// Scanning mode determines speed vs thoroughness tradeoff
    pub scan_mode: ScanMode,
    
    /// Maximum file size to scan (in bytes)
    pub max_file_size: usize,
    
    /// Number of worker threads (0 = auto-detect)
    pub worker_threads: usize,
    
    /// Enable memory mapping for large files
    pub use_mmap: bool,
    
    /// Cache configuration
    pub enable_cache: bool,
    pub cache_size_mb: usize,
    
    /// Early termination
    pub max_critical_findings: Option<usize>,
    pub timeout_seconds: Option<u64>,
    
    /// File filtering
    pub skip_gitignored: bool,
    pub priority_extensions: Vec<String>,
    
    /// Pattern configuration
    pub pattern_sets: Vec<String>,
}

/// Scanning modes with different speed/accuracy tradeoffs
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ScanMode {
    /// Ultra-fast: Critical files only (.env, configs), basic patterns
    Lightning,
    
    /// Fast: Smart sampling, priority patterns, skip large files
    Fast,
    
    /// Balanced: Good coverage with performance optimizations
    Balanced,
    
    /// Thorough: Full scan with all patterns (still optimized)
    Thorough,
    
    /// Paranoid: Everything including experimental patterns
    Paranoid,
}

impl Default for TurboConfig {
    fn default() -> Self {
        Self {
            scan_mode: ScanMode::Balanced,
            max_file_size: 10 * 1024 * 1024, // 10MB
            worker_threads: 0, // Auto-detect
            use_mmap: true,
            enable_cache: true,
            cache_size_mb: 100,
            max_critical_findings: None,
            timeout_seconds: None,
            skip_gitignored: true,
            priority_extensions: vec![
                "env".to_string(),
                "key".to_string(),
                "pem".to_string(),
                "json".to_string(),
                "yml".to_string(),
                "yaml".to_string(),
                "toml".to_string(),
                "ini".to_string(),
                "conf".to_string(),
                "config".to_string(),
            ],
            pattern_sets: vec!["default".to_string()],
        }
    }
}

/// High-performance security analyzer
pub struct TurboSecurityAnalyzer {
    config: TurboConfig,
    pattern_engine: Arc<PatternEngine>,
    cache: Arc<SecurityCache>,
    file_discovery: Arc<FileDiscovery>,
}

impl TurboSecurityAnalyzer {
    /// Create a new turbo security analyzer
    pub fn new(config: TurboConfig) -> Result<Self, SecurityError> {
        let start = Instant::now();
        
        // Initialize pattern engine with compiled patterns
        let pattern_engine = Arc::new(PatternEngine::new(&config)?);
        info!("Pattern engine initialized with {} patterns in {:?}", 
              pattern_engine.pattern_count(), start.elapsed());
        
        // Initialize cache
        let cache = Arc::new(SecurityCache::new(config.cache_size_mb));
        
        // Initialize file discovery
        let discovery_config = DiscoveryConfig {
            use_git: config.skip_gitignored,
            max_file_size: config.max_file_size,
            priority_extensions: config.priority_extensions.clone(),
            scan_mode: config.scan_mode,
        };
        let file_discovery = Arc::new(FileDiscovery::new(discovery_config));
        
        Ok(Self {
            config,
            pattern_engine,
            cache,
            file_discovery,
        })
    }
    
    /// Analyze a project with turbo performance
    pub fn analyze_project(&self, project_root: &Path) -> Result<SecurityReport, SecurityError> {
        let start = Instant::now();
        info!("🚀 Starting turbo security analysis for: {}", project_root.display());
        
        // Phase 1: Ultra-fast file discovery
        let discovery_start = Instant::now();
        let files = self.file_discovery.discover_files(project_root)?;
        info!("📁 Discovered {} files in {:?}", files.len(), discovery_start.elapsed());
        
        // Early exit if no files
        if files.is_empty() {
            return Ok(SecurityReport::empty());
        }
        
        // Phase 2: Intelligent filtering and prioritization
        let filtered_files = self.filter_and_prioritize_files(files);
        info!("🎯 Filtered to {} high-priority files", filtered_files.len());
        
        // Phase 3: Parallel scanning with work-stealing
        let scan_start = Instant::now();
        let (findings, files_scanned) = self.parallel_scan(filtered_files)?;
        info!("🔍 Scanned files in {:?}, found {} findings",
              scan_start.elapsed(), findings.len());

        // Phase 4: Result aggregation and report generation
        let report = ResultAggregator::aggregate(findings, start.elapsed(), files_scanned);
        
        info!("✅ Turbo analysis completed in {:?}", start.elapsed());
        Ok(report)
    }
    
    /// Filter and prioritize files based on scan mode and heuristics
    fn filter_and_prioritize_files(&self, files: Vec<FileMetadata>) -> Vec<FileMetadata> {
        use ScanMode::*;
        
        let mut filtered: Vec<FileMetadata> = match self.config.scan_mode {
            Lightning => {
                // Ultra-fast: Only critical files
                files.into_iter()
                    .filter(|f| f.is_critical())
                    .take(100) // Hard limit for speed
                    .collect()
            }
            Fast => {
                // Fast: Priority files + sample of others
                let (priority, others): (Vec<_>, Vec<_>) = files.into_iter()
                    .partition(|f| f.is_priority());
                
                let mut result = priority;
                // Sample 20% of other files
                let sample_size = others.len() / 5;
                result.extend(others.into_iter().take(sample_size));
                result
            }
            Balanced => {
                // Balanced: All priority files + 50% of others
                let (priority, others): (Vec<_>, Vec<_>) = files.into_iter()
                    .partition(|f| f.is_priority());
                
                let mut result = priority;
                let sample_size = others.len() / 2;
                result.extend(others.into_iter().take(sample_size));
                result
            }
            Thorough => {
                // Thorough: All files except huge ones
                files.into_iter()
                    .filter(|f| f.size < self.config.max_file_size)
                    .collect()
            }
            Paranoid => {
                // Paranoid: Everything
                files
            }
        };
        
        // Sort by priority score (critical files first)
        filtered.par_sort_by_key(|f| std::cmp::Reverse(f.priority_score()));
        filtered
    }
    
    /// Parallel scan with work-stealing and early termination
    fn parallel_scan(&self, files: Vec<FileMetadata>) -> Result<(Vec<SecurityFinding>, usize), SecurityError> {
        let thread_count = if self.config.worker_threads == 0 {
            num_cpus::get()
        } else {
            self.config.worker_threads
        };
        
        // Create channels for work distribution
        let (task_sender, task_receiver) = bounded::<ScanTask>(thread_count * 10);
        let (result_sender, result_receiver) = bounded::<ScanResult>(thread_count * 10);
        
        // Atomic counter for early termination
        let critical_count = Arc::new(parking_lot::Mutex::new(0));
        let should_terminate = Arc::new(parking_lot::RwLock::new(false));
        
        // Spawn scanner threads
        let scanner_handles: Vec<_> = (0..thread_count)
            .map(|thread_id| {
                let scanner = FileScanner::new(
                    thread_id,
                    Arc::clone(&self.pattern_engine),
                    Arc::clone(&self.cache),
                    self.config.use_mmap,
                );
                
                let task_receiver = task_receiver.clone();
                let result_sender = result_sender.clone();
                let critical_count = Arc::clone(&critical_count);
                let should_terminate = Arc::clone(&should_terminate);
                let max_critical = self.config.max_critical_findings;
                
                std::thread::spawn(move || {
                    scanner.run(
                        task_receiver,
                        result_sender,
                        critical_count,
                        should_terminate,
                        max_critical,
                    )
                })
            })
            .collect();
        
        // Drop original receiver to signal completion
        drop(task_receiver);
        
        // Send scan tasks
        let task_sender_thread = {
            let task_sender = task_sender.clone();
            let should_terminate = Arc::clone(&should_terminate);
            
            std::thread::spawn(move || {
                for (idx, file) in files.into_iter().enumerate() {
                    // Check for early termination
                    if *should_terminate.read() {
                        debug!("Early termination triggered, stopping task distribution");
                        break;
                    }
                    
                    let task = ScanTask {
                        id: idx,
                        file,
                        quick_reject: idx > 1000, // Quick reject for files after first 1000
                    };
                    
                    if task_sender.send(task).is_err() {
                        break; // Channel closed
                    }
                }
            })
        };
        
        // Drop original sender to signal completion
        drop(task_sender);
        drop(result_sender);
        
        // Collect results
        let mut all_findings = Vec::new();
        let mut files_scanned = 0;
        let mut files_skipped = 0;
        
        while let Ok(result) = result_receiver.recv() {
            match result {
                ScanResult::Findings(findings) => {
                    all_findings.extend(findings);
                    files_scanned += 1;
                }
                ScanResult::Skipped => {
                    files_skipped += 1;
                }
                ScanResult::Error(err) => {
                    debug!("Scan error: {}", err);
                }
            }
            
            // Progress reporting every 100 files
            if (files_scanned + files_skipped) % 100 == 0 {
                trace!("Progress: {} scanned, {} skipped", files_scanned, files_skipped);
            }
        }
        
        // Wait for threads to complete
        task_sender_thread.join().unwrap();
        for handle in scanner_handles {
            handle.join().unwrap();
        }
        
        info!("Scan complete: {} files scanned, {} skipped, {} findings",
              files_scanned, files_skipped, all_findings.len());

        Ok((all_findings, files_scanned))
    }
}

#[derive(Debug, thiserror::Error)]
pub enum SecurityError {
    #[error("Pattern engine error: {0}")]
    PatternEngine(String),
    
    #[error("File discovery error: {0}")]
    FileDiscovery(String),
    
    #[error("IO error: {0}")]
    Io(#[from] std::io::Error),
    
    #[error("Cache error: {0}")]
    Cache(String),
}

#[cfg(test)]
mod tests {
    use super::*;
    use tempfile::TempDir;
    use std::fs;
    
    #[test]
    fn test_turbo_analyzer_creation() {
        let config = TurboConfig::default();
        let analyzer = TurboSecurityAnalyzer::new(config);
        assert!(analyzer.is_ok());
    }
    
    #[test]
    fn test_scan_modes() {
        let temp_dir = TempDir::new().unwrap();
        
        // Create test files
        fs::write(temp_dir.path().join(".env"), "API_KEY=secret123").unwrap();
        fs::write(temp_dir.path().join("config.json"), r#"{"key": "value"}"#).unwrap();
        fs::write(temp_dir.path().join("main.rs"), "fn main() {}").unwrap();
        
        // Test Lightning mode (should only scan critical files)
        let mut config = TurboConfig::default();
        config.scan_mode = ScanMode::Lightning;
        
        let analyzer = TurboSecurityAnalyzer::new(config).unwrap();
        let report = analyzer.analyze_project(temp_dir.path()).unwrap();
        
        // Should find the .env file
        assert!(report.total_findings > 0);
    }
}