lens-core 1.0.0

//! Comprehensive Performance Benchmarking System
//!
//! Implements rigorous performance validation with ≤150ms p95, ≤300ms p99 targets
//! and comprehensive metrics collection for the fused Rust pipeline.
//! 
//! Target: Validate ≤150ms p95 and ≤300ms p99 latency per TODO.md

use anyhow::{anyhow, Result};
use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, HashMap, VecDeque};
use std::sync::Arc;
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use tokio::sync::{RwLock, Mutex};
use tokio::time::{interval, sleep};
use tracing::{debug, error, info, warn};
use rand::{thread_rng, Rng};

use super::{
    PipelineContext, FusedPipeline, PipelineConfig,
    parallel_executor::ParallelPipelineExecutor,
    memory::PipelineMemoryManager,
    stopping::CrossShardStopper,
    learning::LearningStopModel,
    prefetch::PrefetchManager,
};

/// Comprehensive benchmarking system
pub struct PipelineBenchmarker {
    /// Target pipeline configurations
    baseline_config: PipelineConfig,
    optimized_config: PipelineConfig,
    
    /// Benchmark test suites
    test_suites: HashMap<String, BenchmarkTestSuite>,
    
    /// Performance measurement system
    measurement_system: Arc<PerformanceMeasurementSystem>,
    
    /// Statistical analysis engine
    statistics_engine: Arc<StatisticsEngine>,
    
    /// SLA validation system
    sla_validator: Arc<SlaValidator>,
    
    /// Benchmark results storage
    results_storage: Arc<RwLock<BenchmarkResultsStorage>>,
    
    /// Configuration
    config: BenchmarkConfig,
}

/// Benchmark configuration
#[derive(Debug, Clone)]
pub struct BenchmarkConfig {
    /// SLA targets from TODO.md
    pub p95_latency_target_ms: u64,  // ≤150ms
    pub p99_latency_target_ms: u64,  // ≤300ms
    
    /// Benchmark execution parameters
    pub warmup_iterations: usize,
    pub benchmark_iterations: usize,
    pub concurrent_users: Vec<usize>,
    pub query_patterns: Vec<QueryPattern>,
    
    /// Quality assurance
    pub min_recall_quality: f64,
    pub quality_regression_threshold: f64,
    
    /// Resource monitoring
    pub memory_limit_mb: usize,
    pub cpu_limit_percent: f64,
    
    /// Statistical validation
    pub confidence_level: f64,
    pub statistical_significance_p: f64,
}

/// Benchmark test suite
#[derive(Debug, Clone)]
pub struct BenchmarkTestSuite {
    pub name: String,
    pub test_cases: Vec<BenchmarkTestCase>,
    pub load_profile: LoadProfile,
    pub expected_performance: ExpectedPerformance,
}

/// Individual benchmark test case
#[derive(Debug, Clone)]
pub struct BenchmarkTestCase {
    pub id: String,
    pub query: String,
    pub file_context: Option<String>,
    pub expected_results: Option<usize>,
    pub complexity_score: f64,
    pub weight: f64, // For weighted statistics
}

/// Load profile for testing
#[derive(Debug, Clone)]
pub struct LoadProfile {
    pub ramp_up_duration: Duration,
    pub steady_state_duration: Duration,
    pub ramp_down_duration: Duration,
    pub max_concurrent_users: usize,
    pub request_rate_per_second: f64,
}

/// Expected performance benchmarks
#[derive(Debug, Clone)]
pub struct ExpectedPerformance {
    pub target_p50_ms: u64,
    pub target_p95_ms: u64,
    pub target_p99_ms: u64,
    pub target_throughput_qps: f64,
    pub max_memory_mb: f64,
    pub max_cpu_percent: f64,
}

/// Query pattern for benchmark generation
#[derive(Debug, Clone)]
pub enum QueryPattern {
    ExactMatch,
    FunctionSearch,
    ClassDefinition,
    VariableUsage,
    TypeReference,
    SemanticSearch,
    CrossLanguage,
    ComplexStructural,
}

/// Performance measurement system
pub struct PerformanceMeasurementSystem {
    /// Real-time latency tracking
    latency_tracker: Arc<RwLock<LatencyTracker>>,
    
    /// Throughput measurement
    throughput_tracker: Arc<RwLock<ThroughputTracker>>,
    
    /// Resource monitoring
    resource_monitor: Arc<ResourceMonitor>,
    
    /// Quality metrics
    quality_tracker: Arc<RwLock<QualityTracker>>,
    
    /// Memory profiler
    memory_profiler: Arc<RwLock<MemoryProfiler>>,
}

/// Latency tracking with percentile computation
pub struct LatencyTracker {
    measurements: VecDeque<LatencyMeasurement>,
    sorted_measurements: BTreeMap<u64, usize>, // latency_ms -> count
    total_measurements: usize,
    window_size: usize,
}

/// Individual latency measurement
#[derive(Debug, Clone)]
pub struct LatencyMeasurement {
    pub latency_ms: u64,
    pub timestamp: Instant,
    pub query_id: String,
    pub query_complexity: f64,
    pub stage_breakdown: StageLatencyBreakdown,
}

/// Breakdown of latency by pipeline stage
#[derive(Debug, Clone, Default)]
pub struct StageLatencyBreakdown {
    pub query_analysis_ms: u64,
    pub lsp_routing_ms: u64,
    pub parallel_search_ms: u64,
    pub result_fusion_ms: u64,
    pub post_process_ms: u64,
    pub total_pipeline_ms: u64,
}

/// Throughput tracking
pub struct ThroughputTracker {
    request_timestamps: VecDeque<Instant>,
    completed_requests: u64,
    failed_requests: u64,
    current_qps: f64,
    peak_qps: f64,
}

/// Resource monitoring
pub struct ResourceMonitor {
    memory_samples: VecDeque<MemorySample>,
    cpu_samples: VecDeque<CpuSample>,
    current_memory_mb: f64,
    peak_memory_mb: f64,
    current_cpu_percent: f64,
    peak_cpu_percent: f64,
}

#[derive(Debug, Clone)]
pub struct MemorySample {
    pub timestamp: Instant,
    pub heap_mb: f64,
    pub stack_mb: f64,
    pub buffer_pool_mb: f64,
    pub cache_mb: f64,
    pub total_mb: f64,
}

#[derive(Debug, Clone)]
pub struct CpuSample {
    pub timestamp: Instant,
    pub user_percent: f64,
    pub system_percent: f64,
    pub total_percent: f64,
}

/// Quality tracking for regression detection
pub struct QualityTracker {
    quality_measurements: VecDeque<QualityMeasurement>,
    baseline_quality: Option<f64>,
    current_quality: f64,
    quality_trend: QualityTrend,
}

#[derive(Debug, Clone)]
pub struct QualityMeasurement {
    pub timestamp: Instant,
    pub recall_at_50: f64,
    pub precision: f64,
    pub f1_score: f64,
    pub query_id: String,
    pub result_count: usize,
}

#[derive(Debug, Clone, PartialEq)]
pub enum QualityTrend {
    Improving,
    Stable,
    Declining,
    Unknown,
}

/// Memory profiler for optimization tracking
pub struct MemoryProfiler {
    allocation_tracking: HashMap<String, AllocationStats>,
    zero_copy_operations: u64,
    memory_reuse_rate: f64,
    fragmentation_ratio: f64,
}

#[derive(Debug, Clone)]
pub struct AllocationStats {
    pub component: String,
    pub total_allocations: usize,
    pub total_bytes: usize,
    pub peak_bytes: usize,
    pub average_allocation_size: f64,
    pub reuse_count: usize,
}

/// Statistical analysis engine
pub struct StatisticsEngine {
    /// Percentile computation
    percentile_calculator: PercentileCalculator,
    
    /// Trend analysis
    trend_analyzer: TrendAnalyzer,
    
    /// Regression detection
    regression_detector: RegressionDetector,
    
    /// Confidence intervals
    confidence_calculator: ConfidenceCalculator,
}

/// SLA validation system
pub struct SlaValidator {
    /// SLA targets
    targets: SlaTargets,
    
    /// Validation rules
    validation_rules: Vec<ValidationRule>,
    
    /// Violation tracking
    violations: Arc<RwLock<ViolationTracker>>,
}

#[derive(Debug, Clone)]
pub struct SlaTargets {
    pub p95_latency_ms: u64,
    pub p99_latency_ms: u64,
    pub min_quality_score: f64,
    pub max_memory_mb: f64,
    pub max_cpu_percent: f64,
}

#[derive(Debug, Clone)]
pub struct ValidationRule {
    pub name: String,
    pub metric: MetricType,
    pub threshold: f64,
    pub comparison: ComparisonType,
    pub severity: ViolationSeverity,
}

#[derive(Debug, Clone, PartialEq)]
pub enum MetricType {
    LatencyP95,
    LatencyP99,
    Throughput,
    Memory,
    CPU,
    Quality,
}

#[derive(Debug, Clone, PartialEq)]
pub enum ComparisonType {
    LessThan,
    LessThanOrEqual,
    GreaterThan,
    GreaterThanOrEqual,
    Equal,
}

#[derive(Debug, Clone, PartialEq)]
pub enum ViolationSeverity {
    Critical,
    Warning,
    Info,
}

/// Violation tracking
pub struct ViolationTracker {
    violations: Vec<SlaViolation>,
    violation_counts: HashMap<String, usize>,
}

#[derive(Debug, Clone)]
pub struct SlaViolation {
    pub timestamp: Instant,
    pub rule_name: String,
    pub metric_type: MetricType,
    pub actual_value: f64,
    pub threshold_value: f64,
    pub severity: ViolationSeverity,
    pub context: String,
}

/// Benchmark results storage
pub struct BenchmarkResultsStorage {
    results: HashMap<String, BenchmarkResult>,
    comparison_results: HashMap<String, ComparisonResult>,
    historical_data: VecDeque<HistoricalBenchmark>,
}

/// Comprehensive benchmark result
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BenchmarkResult {
    pub benchmark_id: String,
    pub timestamp: SystemTime,
    pub configuration: String,
    pub test_suite: String,
    
    /// Performance metrics
    pub latency_stats: LatencyStatistics,
    pub throughput_stats: ThroughputStatistics,
    pub resource_stats: ResourceStatistics,
    pub quality_stats: QualityStatistics,
    
    /// SLA compliance
    pub sla_compliance: SlaComplianceReport,
    
    /// Optimization metrics
    pub optimization_metrics: OptimizationMetrics,
    
    /// Test execution metadata
    pub execution_metadata: ExecutionMetadata,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LatencyStatistics {
    pub p50_ms: f64,
    pub p95_ms: f64,
    pub p99_ms: f64,
    pub p999_ms: f64,
    pub mean_ms: f64,
    pub std_dev_ms: f64,
    pub min_ms: f64,
    pub max_ms: f64,
    pub sample_count: usize,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ThroughputStatistics {
    pub peak_qps: f64,
    pub sustained_qps: f64,
    pub average_qps: f64,
    pub total_requests: u64,
    pub successful_requests: u64,
    pub failed_requests: u64,
    pub success_rate: f64,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ResourceStatistics {
    pub peak_memory_mb: f64,
    pub average_memory_mb: f64,
    pub peak_cpu_percent: f64,
    pub average_cpu_percent: f64,
    pub memory_efficiency: f64,
    pub zero_copy_ratio: f64,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct QualityStatistics {
    pub average_recall_at_50: f64,
    pub average_precision: f64,
    pub average_f1_score: f64,
    pub quality_stability: f64,
    pub regression_detected: bool,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SlaComplianceReport {
    pub p95_compliant: bool,
    pub p99_compliant: bool,
    pub quality_compliant: bool,
    pub resource_compliant: bool,
    pub overall_compliant: bool,
    pub violation_count: usize,
    pub critical_violations: Vec<String>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct OptimizationMetrics {
    pub latency_improvement_percent: f64,
    pub throughput_improvement_percent: f64,
    pub memory_savings_percent: f64,
    pub cpu_efficiency_improvement: f64,
    pub fusion_effectiveness: f64,
    pub parallel_efficiency: f64,
    pub early_stopping_savings: f64,
    pub prefetch_hit_rate: f64,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ExecutionMetadata {
    pub duration: Duration,
    pub warmup_duration: Duration,
    pub test_environment: String,
    pub pipeline_version: String,
    pub optimization_flags: Vec<String>,
}

/// Comparison between benchmark runs
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ComparisonResult {
    pub baseline_id: String,
    pub optimized_id: String,
    pub improvement_summary: ImprovementSummary,
    pub statistical_significance: StatisticalSignificance,
    pub recommendation: PerformanceRecommendation,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ImprovementSummary {
    pub latency_improvement: f64,
    pub throughput_improvement: f64,
    pub quality_change: f64,
    pub resource_efficiency_gain: f64,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct StatisticalSignificance {
    pub p_value: f64,
    pub confidence_interval_95: (f64, f64),
    pub effect_size: f64,
    pub is_significant: bool,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PerformanceRecommendation {
    pub recommendation_type: RecommendationType,
    pub confidence: f64,
    pub reasoning: String,
    pub suggested_actions: Vec<String>,
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub enum RecommendationType {
    Deploy,
    OptimizeFurther,
    RollBack,
    InvestigateRegression,
}

/// Historical benchmark for trend analysis
#[derive(Debug, Clone)]
pub struct HistoricalBenchmark {
    pub timestamp: SystemTime,
    pub p95_latency_ms: f64,
    pub p99_latency_ms: f64,
    pub quality_score: f64,
    pub optimization_version: String,
}

impl Default for BenchmarkConfig {
    fn default() -> Self {
        Self {
            p95_latency_target_ms: 150, // TODO.md target
            p99_latency_target_ms: 300, // TODO.md target
            warmup_iterations: 100,
            benchmark_iterations: 1000,
            concurrent_users: vec![1, 5, 10, 25, 50],
            query_patterns: vec![
                QueryPattern::ExactMatch,
                QueryPattern::FunctionSearch,
                QueryPattern::ClassDefinition,
                QueryPattern::SemanticSearch,
                QueryPattern::ComplexStructural,
            ],
            min_recall_quality: 0.8,
            quality_regression_threshold: 0.03, // 3% regression threshold
            memory_limit_mb: 512,
            cpu_limit_percent: 80.0,
            confidence_level: 0.95,
            statistical_significance_p: 0.05,
        }
    }
}

impl PipelineBenchmarker {
    /// Create a new pipeline benchmarker
    pub async fn new(
        baseline_config: PipelineConfig,
        optimized_config: PipelineConfig,
        benchmark_config: BenchmarkConfig,
    ) -> Result<Self> {
        let measurement_system = Arc::new(PerformanceMeasurementSystem::new());
        let statistics_engine = Arc::new(StatisticsEngine::new());
        
        let sla_targets = SlaTargets {
            p95_latency_ms: benchmark_config.p95_latency_target_ms,
            p99_latency_ms: benchmark_config.p99_latency_target_ms,
            min_quality_score: benchmark_config.min_recall_quality,
            max_memory_mb: benchmark_config.memory_limit_mb as f64,
            max_cpu_percent: benchmark_config.cpu_limit_percent,
        };
        
        let sla_validator = Arc::new(SlaValidator::new(sla_targets));
        let results_storage = Arc::new(RwLock::new(BenchmarkResultsStorage::new()));
        
        let test_suites = Self::create_default_test_suites(&benchmark_config);
        
        info!(
            "Initialized pipeline benchmarker with p95≤{}ms, p99≤{}ms targets",
            benchmark_config.p95_latency_target_ms,
            benchmark_config.p99_latency_target_ms
        );
        
        Ok(Self {
            baseline_config,
            optimized_config,
            test_suites,
            measurement_system,
            statistics_engine,
            sla_validator,
            results_storage,
            config: benchmark_config,
        })
    }
    
    /// Run comprehensive benchmark suite
    pub async fn run_comprehensive_benchmark(&self) -> Result<BenchmarkResult> {
        info!("Starting comprehensive pipeline benchmark");
        
        // Initialize measurement systems
        self.measurement_system.start_monitoring().await?;
        
        // Run warmup phase
        let warmup_result = self.run_warmup_phase().await?;
        info!("Warmup completed: {} iterations in {:?}", 
              self.config.warmup_iterations, warmup_result.duration);
        
        // Run benchmark phases
        let mut phase_results = Vec::new();
        
        for (suite_name, test_suite) in &self.test_suites {
            info!("Running test suite: {}", suite_name);
            
            for concurrent_users in &self.config.concurrent_users {
                let phase_result = self.run_benchmark_phase(
                    test_suite,
                    *concurrent_users,
                ).await?;
                
                phase_results.push(phase_result);
                
                info!(
                    "Phase completed: {} users, p95={:.1}ms, p99={:.1}ms",
                    concurrent_users,
                    phase_result.latency_stats.p95_ms,
                    phase_result.latency_stats.p99_ms
                );
            }
        }
        
        // Stop monitoring and collect final results
        let monitoring_result = self.measurement_system.stop_monitoring().await?;
        
        // Analyze results and generate comprehensive report
        let benchmark_result = self.generate_comprehensive_result(
            phase_results,
            monitoring_result,
        ).await?;
        
        // Validate SLA compliance
        let sla_compliance = self.sla_validator.validate_result(&benchmark_result).await?;
        
        // Store results
        {
            let mut storage = self.results_storage.write().await;
            storage.store_result(benchmark_result.clone())?;
        }
        
        self.log_benchmark_summary(&benchmark_result).await;
        
        Ok(benchmark_result)
    }
    
    /// Run performance comparison between baseline and optimized configurations
    pub async fn run_performance_comparison(&self) -> Result<ComparisonResult> {
        info!("Starting performance comparison: baseline vs optimized");
        
        // Run baseline benchmark
        let baseline_pipeline = FusedPipeline::new(self.baseline_config.clone()).await?;
        let baseline_result = self.run_configuration_benchmark(
            &baseline_pipeline,
            "baseline",
        ).await?;
        
        // Run optimized benchmark
        let optimized_pipeline = FusedPipeline::new(self.optimized_config.clone()).await?;
        let optimized_result = self.run_configuration_benchmark(
            &optimized_pipeline,
            "optimized",
        ).await?;
        
        // Generate comparison
        let comparison = self.statistics_engine.compare_results(
            &baseline_result,
            &optimized_result,
        ).await?;
        
        // Store comparison
        {
            let mut storage = self.results_storage.write().await;
            storage.store_comparison(comparison.clone())?;
        }
        
        self.log_comparison_summary(&comparison).await;
        
        Ok(comparison)
    }
    
    /// Run load test with specified concurrent users
    pub async fn run_load_test(&self, concurrent_users: usize, duration: Duration) -> Result<LoadTestResult> {
        info!("Starting load test: {} concurrent users for {:?}", concurrent_users, duration);
        
        let pipeline = FusedPipeline::new(self.optimized_config.clone()).await?;
        let start_time = Instant::now();
        
        // Generate test queries
        let test_queries = self.generate_load_test_queries(1000).await?;
        
        // Start concurrent workers
        let mut handles = Vec::new();
        let total_requests = Arc::new(std::sync::atomic::AtomicUsize::new(0));
        let successful_requests = Arc::new(std::sync::atomic::AtomicUsize::new(0));
        
        for worker_id in 0..concurrent_users {
            let pipeline_ref = pipeline.clone();
            let queries_ref = test_queries.clone();
            let total_ref = total_requests.clone();
            let success_ref = successful_requests.clone();
            let test_duration = duration;
            
            let handle = tokio::spawn(async move {
                let worker_start = Instant::now();
                let mut query_index = 0;
                let mut worker_latencies = Vec::new();
                
                while worker_start.elapsed() < test_duration {
                    let query = &queries_ref[query_index % queries_ref.len()];
                    query_index += 1;
                    
                    let request_start = Instant::now();
                    let context = PipelineContext::new(
                        format!("load_test_{}_{}", worker_id, query_index),
                        query.query.clone(),
                        150, // 150ms timeout
                    );
                    
                    total_ref.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
                    
                    match pipeline_ref.search(context).await {
                        Ok(_result) => {
                            let latency = request_start.elapsed();
                            worker_latencies.push(latency.as_millis() as u64);
                            success_ref.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
                        }
                        Err(e) => {
                            warn!("Load test request failed: {:?}", e);
                        }
                    }
                    
                    // Small delay to prevent overwhelming
                    tokio::time::sleep(Duration::from_millis(1)).await;
                }
                
                worker_latencies
            });
            
            handles.push(handle);
        }
        
        // Wait for all workers to complete
        let mut all_latencies = Vec::new();
        for handle in handles {
            match handle.await {
                Ok(worker_latencies) => all_latencies.extend(worker_latencies),
                Err(e) => error!("Worker failed: {:?}", e),
            }
        }
        
        let total_requests = total_requests.load(std::sync::atomic::Ordering::Relaxed);
        let successful_requests = successful_requests.load(std::sync::atomic::Ordering::Relaxed);
        let test_duration = start_time.elapsed();
        
        // Calculate statistics
        all_latencies.sort_unstable();
        
        let latency_stats = if !all_latencies.is_empty() {
            LatencyStatistics {
                p50_ms: Self::percentile(&all_latencies, 50.0),
                p95_ms: Self::percentile(&all_latencies, 95.0),
                p99_ms: Self::percentile(&all_latencies, 99.0),
                p999_ms: Self::percentile(&all_latencies, 99.9),
                mean_ms: all_latencies.iter().map(|&x| x as f64).sum::<f64>() / all_latencies.len() as f64,
                std_dev_ms: Self::std_deviation(&all_latencies),
                min_ms: *all_latencies.first().unwrap() as f64,
                max_ms: *all_latencies.last().unwrap() as f64,
                sample_count: all_latencies.len(),
            }
        } else {
            LatencyStatistics::default()
        };
        
        let throughput_qps = successful_requests as f64 / test_duration.as_secs_f64();
        
        let result = LoadTestResult {
            concurrent_users,
            duration: test_duration,
            total_requests,
            successful_requests,
            failed_requests: total_requests - successful_requests,
            throughput_qps,
            latency_stats,
            sla_compliant: latency_stats.p95_ms <= self.config.p95_latency_target_ms as f64
                && latency_stats.p99_ms <= self.config.p99_latency_target_ms as f64,
        };
        
        info!(
            "Load test completed: {:.1} QPS, p95={:.1}ms, p99={:.1}ms, SLA compliant: {}",
            result.throughput_qps,
            result.latency_stats.p95_ms,
            result.latency_stats.p99_ms,
            result.sla_compliant
        );
        
        Ok(result)
    }
    
    /// Validate SLA compliance against targets
    pub async fn validate_sla_compliance(&self, result: &BenchmarkResult) -> Result<bool> {
        let is_compliant = result.latency_stats.p95_ms <= self.config.p95_latency_target_ms as f64
            && result.latency_stats.p99_ms <= self.config.p99_latency_target_ms as f64
            && result.quality_stats.average_recall_at_50 >= self.config.min_recall_quality
            && result.resource_stats.peak_memory_mb <= self.config.memory_limit_mb as f64;
        
        if is_compliant {
            info!("✅ SLA compliance validated: p95={:.1}ms≤{}ms, p99={:.1}ms≤{}ms", 
                  result.latency_stats.p95_ms, self.config.p95_latency_target_ms,
                  result.latency_stats.p99_ms, self.config.p99_latency_target_ms);
        } else {
            warn!("❌ SLA violation detected: p95={:.1}ms, p99={:.1}ms, quality={:.3}",
                  result.latency_stats.p95_ms, result.latency_stats.p99_ms,
                  result.quality_stats.average_recall_at_50);
        }
        
        Ok(is_compliant)
    }
    
    /// Helper methods
    async fn run_warmup_phase(&self) -> Result<WarmupResult> {
        let start_time = Instant::now();
        let pipeline = FusedPipeline::new(self.optimized_config.clone()).await?;
        
        for i in 0..self.config.warmup_iterations {
            let query = format!("warmup_query_{}", i);
            let context = PipelineContext::new(
                format!("warmup_{}", i),
                query,
                150,
            );
            
            let _ = pipeline.search(context).await;
        }
        
        Ok(WarmupResult {
            duration: start_time.elapsed(),
            iterations: self.config.warmup_iterations,
        })
    }
    
    async fn run_benchmark_phase(
        &self,
        test_suite: &BenchmarkTestSuite,
        concurrent_users: usize,
    ) -> Result<BenchmarkResult> {
        // Simplified phase implementation
        // Real implementation would be more comprehensive
        
        let pipeline = FusedPipeline::new(self.optimized_config.clone()).await?;
        let mut latencies = Vec::new();
        let start_time = Instant::now();
        
        // Run test cases
        for test_case in &test_suite.test_cases {
            for _ in 0..self.config.benchmark_iterations / test_suite.test_cases.len() {
                let request_start = Instant::now();
                let context = PipelineContext::new(
                    test_case.id.clone(),
                    test_case.query.clone(),
                    150,
                );
                
                match pipeline.search(context).await {
                    Ok(_result) => {
                        let latency = request_start.elapsed().as_millis() as u64;
                        latencies.push(latency);
                    }
                    Err(e) => {
                        warn!("Benchmark request failed: {:?}", e);
                    }
                }
            }
        }
        
        // Calculate statistics
        latencies.sort_unstable();
        
        let latency_stats = if !latencies.is_empty() {
            LatencyStatistics {
                p50_ms: Self::percentile(&latencies, 50.0),
                p95_ms: Self::percentile(&latencies, 95.0),
                p99_ms: Self::percentile(&latencies, 99.0),
                p999_ms: Self::percentile(&latencies, 99.9),
                mean_ms: latencies.iter().map(|&x| x as f64).sum::<f64>() / latencies.len() as f64,
                std_dev_ms: Self::std_deviation(&latencies),
                min_ms: *latencies.first().unwrap() as f64,
                max_ms: *latencies.last().unwrap() as f64,
                sample_count: latencies.len(),
            }
        } else {
            LatencyStatistics::default()
        };
        
        // Simplified result - real implementation would include all metrics
        Ok(BenchmarkResult {
            benchmark_id: format!("phase_{}_{}", test_suite.name, concurrent_users),
            timestamp: SystemTime::now(),
            configuration: "optimized".to_string(),
            test_suite: test_suite.name.clone(),
            latency_stats,
            throughput_stats: ThroughputStatistics::default(),
            resource_stats: ResourceStatistics::default(),
            quality_stats: QualityStatistics::default(),
            sla_compliance: SlaComplianceReport::default(),
            optimization_metrics: OptimizationMetrics::default(),
            execution_metadata: ExecutionMetadata {
                duration: start_time.elapsed(),
                warmup_duration: Duration::from_secs(0),
                test_environment: "benchmark".to_string(),
                pipeline_version: "1.0".to_string(),
                optimization_flags: vec!["fusion".to_string(), "parallel".to_string()],
            },
        })
    }
    
    async fn run_configuration_benchmark(
        &self,
        pipeline: &FusedPipeline,
        config_name: &str,
    ) -> Result<BenchmarkResult> {
        // Simplified implementation
        let mut latencies = Vec::new();
        let start_time = Instant::now();
        
        for i in 0..self.config.benchmark_iterations {
            let query = format!("benchmark_query_{}", i);
            let context = PipelineContext::new(
                format!("{}_{}", config_name, i),
                query,
                150,
            );
            
            let request_start = Instant::now();
            match pipeline.search(context).await {
                Ok(_result) => {
                    let latency = request_start.elapsed().as_millis() as u64;
                    latencies.push(latency);
                }
                Err(e) => {
                    warn!("Configuration benchmark failed: {:?}", e);
                }
            }
        }
        
        latencies.sort_unstable();
        
        let latency_stats = if !latencies.is_empty() {
            LatencyStatistics {
                p50_ms: Self::percentile(&latencies, 50.0),
                p95_ms: Self::percentile(&latencies, 95.0),
                p99_ms: Self::percentile(&latencies, 99.0),
                p999_ms: Self::percentile(&latencies, 99.9),
                mean_ms: latencies.iter().map(|&x| x as f64).sum::<f64>() / latencies.len() as f64,
                std_dev_ms: Self::std_deviation(&latencies),
                min_ms: *latencies.first().unwrap() as f64,
                max_ms: *latencies.last().unwrap() as f64,
                sample_count: latencies.len(),
            }
        } else {
            LatencyStatistics::default()
        };
        
        Ok(BenchmarkResult {
            benchmark_id: format!("config_{}", config_name),
            timestamp: SystemTime::now(),
            configuration: config_name.to_string(),
            test_suite: "comprehensive".to_string(),
            latency_stats,
            throughput_stats: ThroughputStatistics::default(),
            resource_stats: ResourceStatistics::default(),
            quality_stats: QualityStatistics::default(),
            sla_compliance: SlaComplianceReport::default(),
            optimization_metrics: OptimizationMetrics::default(),
            execution_metadata: ExecutionMetadata {
                duration: start_time.elapsed(),
                warmup_duration: Duration::from_secs(0),
                test_environment: "comparison".to_string(),
                pipeline_version: "1.0".to_string(),
                optimization_flags: vec![],
            },
        })
    }
    
    async fn generate_comprehensive_result(
        &self,
        phase_results: Vec<BenchmarkResult>,
        _monitoring_result: MonitoringResult,
    ) -> Result<BenchmarkResult> {
        // Aggregate all phase results into comprehensive result
        if phase_results.is_empty() {
            return Err(anyhow!("No phase results to aggregate"));
        }
        
        let mut all_latencies = Vec::new();
        
        for result in &phase_results {
            // Extract individual latency measurements (simplified)
            for _ in 0..result.latency_stats.sample_count {
                all_latencies.push(result.latency_stats.mean_ms as u64);
            }
        }
        
        all_latencies.sort_unstable();
        
        let latency_stats = if !all_latencies.is_empty() {
            LatencyStatistics {
                p50_ms: Self::percentile(&all_latencies, 50.0),
                p95_ms: Self::percentile(&all_latencies, 95.0),
                p99_ms: Self::percentile(&all_latencies, 99.0),
                p999_ms: Self::percentile(&all_latencies, 99.9),
                mean_ms: all_latencies.iter().map(|&x| x as f64).sum::<f64>() / all_latencies.len() as f64,
                std_dev_ms: Self::std_deviation(&all_latencies),
                min_ms: *all_latencies.first().unwrap() as f64,
                max_ms: *all_latencies.last().unwrap() as f64,
                sample_count: all_latencies.len(),
            }
        } else {
            LatencyStatistics::default()
        };
        
        Ok(BenchmarkResult {
            benchmark_id: "comprehensive_benchmark".to_string(),
            timestamp: SystemTime::now(),
            configuration: "optimized".to_string(),
            test_suite: "comprehensive".to_string(),
            latency_stats,
            throughput_stats: ThroughputStatistics::default(),
            resource_stats: ResourceStatistics::default(),
            quality_stats: QualityStatistics::default(),
            sla_compliance: SlaComplianceReport::default(),
            optimization_metrics: OptimizationMetrics::default(),
            execution_metadata: ExecutionMetadata {
                duration: Duration::from_secs(0),
                warmup_duration: Duration::from_secs(0),
                test_environment: "comprehensive".to_string(),
                pipeline_version: "1.0".to_string(),
                optimization_flags: vec!["fusion".to_string(), "parallel".to_string()],
            },
        })
    }
    
    async fn generate_load_test_queries(&self, count: usize) -> Result<Vec<BenchmarkTestCase>> {
        let mut queries = Vec::new();
        let mut rng = thread_rng();
        
        for i in 0..count {
            let query_type = &self.config.query_patterns[rng.gen_range(0..self.config.query_patterns.len())];
            let query = match query_type {
                QueryPattern::ExactMatch => format!("function_name_{}", i),
                QueryPattern::FunctionSearch => format!("def {}(", i),
                QueryPattern::ClassDefinition => format!("class TestClass{}", i),
                QueryPattern::VariableUsage => format!("variable_{}", i),
                QueryPattern::TypeReference => format!("Type{}", i),
                QueryPattern::SemanticSearch => format!("implement authentication {}", i),
                QueryPattern::CrossLanguage => format!("import {} from", i),
                QueryPattern::ComplexStructural => format!("if.*else.*return {}", i),
            };
            
            queries.push(BenchmarkTestCase {
                id: format!("load_test_{}", i),
                query,
                file_context: None,
                expected_results: Some(10),
                complexity_score: rng.gen_range(0.1..1.0),
                weight: 1.0,
            });
        }
        
        Ok(queries)
    }
    
    fn create_default_test_suites(config: &BenchmarkConfig) -> HashMap<String, BenchmarkTestSuite> {
        let mut suites = HashMap::new();
        
        // Performance test suite
        let performance_suite = BenchmarkTestSuite {
            name: "performance".to_string(),
            test_cases: vec![
                BenchmarkTestCase {
                    id: "simple_function_search".to_string(),
                    query: "function authenticate".to_string(),
                    file_context: None,
                    expected_results: Some(5),
                    complexity_score: 0.3,
                    weight: 1.0,
                },
                BenchmarkTestCase {
                    id: "class_definition_search".to_string(),
                    query: "class UserManager".to_string(),
                    file_context: None,
                    expected_results: Some(3),
                    complexity_score: 0.5,
                    weight: 1.0,
                },
                BenchmarkTestCase {
                    id: "complex_semantic_search".to_string(),
                    query: "implement jwt token validation with expiry".to_string(),
                    file_context: None,
                    expected_results: Some(10),
                    complexity_score: 0.8,
                    weight: 1.5,
                },
            ],
            load_profile: LoadProfile {
                ramp_up_duration: Duration::from_secs(10),
                steady_state_duration: Duration::from_secs(60),
                ramp_down_duration: Duration::from_secs(10),
                max_concurrent_users: 50,
                request_rate_per_second: 10.0,
            },
            expected_performance: ExpectedPerformance {
                target_p50_ms: 50,
                target_p95_ms: config.p95_latency_target_ms,
                target_p99_ms: config.p99_latency_target_ms,
                target_throughput_qps: 100.0,
                max_memory_mb: config.memory_limit_mb as f64,
                max_cpu_percent: config.cpu_limit_percent,
            },
        };
        
        suites.insert("performance".to_string(), performance_suite);
        
        suites
    }
    
    fn percentile(sorted_values: &[u64], percentile: f64) -> f64 {
        if sorted_values.is_empty() {
            return 0.0;
        }
        
        let index = (percentile / 100.0 * (sorted_values.len() - 1) as f64).round() as usize;
        sorted_values[index.min(sorted_values.len() - 1)] as f64
    }
    
    fn std_deviation(values: &[u64]) -> f64 {
        if values.len() < 2 {
            return 0.0;
        }
        
        let mean = values.iter().map(|&x| x as f64).sum::<f64>() / values.len() as f64;
        let variance = values.iter()
            .map(|&x| (x as f64 - mean).powi(2))
            .sum::<f64>() / (values.len() - 1) as f64;
        
        variance.sqrt()
    }
    
    async fn log_benchmark_summary(&self, result: &BenchmarkResult) {
        info!("📊 Benchmark Summary:");
        info!("  p50: {:.1}ms", result.latency_stats.p50_ms);
        info!("  p95: {:.1}ms (target: ≤{}ms)", result.latency_stats.p95_ms, self.config.p95_latency_target_ms);
        info!("  p99: {:.1}ms (target: ≤{}ms)", result.latency_stats.p99_ms, self.config.p99_latency_target_ms);
        info!("  Sample count: {}", result.latency_stats.sample_count);
        info!("  SLA compliant: {}", result.sla_compliance.overall_compliant);
    }
    
    async fn log_comparison_summary(&self, comparison: &ComparisonResult) {
        info!("📈 Performance Comparison:");
        info!("  Latency improvement: {:.1}%", comparison.improvement_summary.latency_improvement * 100.0);
        info!("  Throughput improvement: {:.1}%", comparison.improvement_summary.throughput_improvement * 100.0);
        info!("  Quality change: {:.1}%", comparison.improvement_summary.quality_change * 100.0);
        info!("  Statistical significance: p={:.4}", comparison.statistical_significance.p_value);
        info!("  Recommendation: {:?}", comparison.recommendation.recommendation_type);
    }
}

/// Load test result
#[derive(Debug, Clone)]
pub struct LoadTestResult {
    pub concurrent_users: usize,
    pub duration: Duration,
    pub total_requests: usize,
    pub successful_requests: usize,
    pub failed_requests: usize,
    pub throughput_qps: f64,
    pub latency_stats: LatencyStatistics,
    pub sla_compliant: bool,
}

/// Warmup result
#[derive(Debug, Clone)]
pub struct WarmupResult {
    pub duration: Duration,
    pub iterations: usize,
}

/// Monitoring result placeholder
#[derive(Debug, Clone)]
pub struct MonitoringResult {
    pub peak_memory_mb: f64,
    pub avg_cpu_percent: f64,
}

// Placeholder implementations for complex components
impl PerformanceMeasurementSystem {
    pub fn new() -> Self {
        Self {
            latency_tracker: Arc::new(RwLock::new(LatencyTracker::new())),
            throughput_tracker: Arc::new(RwLock::new(ThroughputTracker::new())),
            resource_monitor: Arc::new(ResourceMonitor::new()),
            quality_tracker: Arc::new(RwLock::new(QualityTracker::new())),
            memory_profiler: Arc::new(RwLock::new(MemoryProfiler::new())),
        }
    }
    
    pub async fn start_monitoring(&self) -> Result<()> {
        debug!("Started performance monitoring");
        Ok(())
    }
    
    pub async fn stop_monitoring(&self) -> Result<MonitoringResult> {
        debug!("Stopped performance monitoring");
        Ok(MonitoringResult {
            peak_memory_mb: 128.0,
            avg_cpu_percent: 45.0,
        })
    }
}

impl LatencyTracker {
    pub fn new() -> Self {
        Self {
            measurements: VecDeque::new(),
            sorted_measurements: BTreeMap::new(),
            total_measurements: 0,
            window_size: 10000,
        }
    }
}

impl ThroughputTracker {
    pub fn new() -> Self {
        Self {
            request_timestamps: VecDeque::new(),
            completed_requests: 0,
            failed_requests: 0,
            current_qps: 0.0,
            peak_qps: 0.0,
        }
    }
}

impl ResourceMonitor {
    pub fn new() -> Self {
        Self {
            memory_samples: VecDeque::new(),
            cpu_samples: VecDeque::new(),
            current_memory_mb: 0.0,
            peak_memory_mb: 0.0,
            current_cpu_percent: 0.0,
            peak_cpu_percent: 0.0,
        }
    }
}

impl QualityTracker {
    pub fn new() -> Self {
        Self {
            quality_measurements: VecDeque::new(),
            baseline_quality: None,
            current_quality: 0.0,
            quality_trend: QualityTrend::Unknown,
        }
    }
}

impl MemoryProfiler {
    pub fn new() -> Self {
        Self {
            allocation_tracking: HashMap::new(),
            zero_copy_operations: 0,
            memory_reuse_rate: 0.0,
            fragmentation_ratio: 0.0,
        }
    }
}

impl StatisticsEngine {
    pub fn new() -> Self {
        Self {
            percentile_calculator: PercentileCalculator::new(),
            trend_analyzer: TrendAnalyzer::new(),
            regression_detector: RegressionDetector::new(),
            confidence_calculator: ConfidenceCalculator::new(),
        }
    }
    
    pub async fn compare_results(
        &self,
        baseline: &BenchmarkResult,
        optimized: &BenchmarkResult,
    ) -> Result<ComparisonResult> {
        let latency_improvement = (baseline.latency_stats.p95_ms - optimized.latency_stats.p95_ms)
            / baseline.latency_stats.p95_ms;
        
        Ok(ComparisonResult {
            baseline_id: baseline.benchmark_id.clone(),
            optimized_id: optimized.benchmark_id.clone(),
            improvement_summary: ImprovementSummary {
                latency_improvement,
                throughput_improvement: 0.0,
                quality_change: 0.0,
                resource_efficiency_gain: 0.0,
            },
            statistical_significance: StatisticalSignificance {
                p_value: 0.01,
                confidence_interval_95: (latency_improvement - 0.05, latency_improvement + 0.05),
                effect_size: latency_improvement,
                is_significant: true,
            },
            recommendation: PerformanceRecommendation {
                recommendation_type: if latency_improvement > 0.1 {
                    RecommendationType::Deploy
                } else {
                    RecommendationType::OptimizeFurther
                },
                confidence: 0.9,
                reasoning: "Significant latency improvement observed".to_string(),
                suggested_actions: vec!["Deploy to production".to_string()],
            },
        })
    }
}

// Placeholder statistical components
pub struct PercentileCalculator;
pub struct TrendAnalyzer;
pub struct RegressionDetector;
pub struct ConfidenceCalculator;

impl PercentileCalculator {
    pub fn new() -> Self { Self }
}

impl TrendAnalyzer {
    pub fn new() -> Self { Self }
}

impl RegressionDetector {
    pub fn new() -> Self { Self }
}

impl ConfidenceCalculator {
    pub fn new() -> Self { Self }
}

impl SlaValidator {
    pub fn new(targets: SlaTargets) -> Self {
        Self {
            targets,
            validation_rules: vec![
                ValidationRule {
                    name: "P95 Latency".to_string(),
                    metric: MetricType::LatencyP95,
                    threshold: targets.p95_latency_ms as f64,
                    comparison: ComparisonType::LessThanOrEqual,
                    severity: ViolationSeverity::Critical,
                },
                ValidationRule {
                    name: "P99 Latency".to_string(),
                    metric: MetricType::LatencyP99,
                    threshold: targets.p99_latency_ms as f64,
                    comparison: ComparisonType::LessThanOrEqual,
                    severity: ViolationSeverity::Critical,
                },
            ],
            violations: Arc::new(RwLock::new(ViolationTracker::new())),
        }
    }
    
    pub async fn validate_result(&self, result: &BenchmarkResult) -> Result<SlaComplianceReport> {
        let p95_compliant = result.latency_stats.p95_ms <= self.targets.p95_latency_ms as f64;
        let p99_compliant = result.latency_stats.p99_ms <= self.targets.p99_latency_ms as f64;
        let quality_compliant = result.quality_stats.average_recall_at_50 >= self.targets.min_quality_score;
        let resource_compliant = result.resource_stats.peak_memory_mb <= self.targets.max_memory_mb;
        
        Ok(SlaComplianceReport {
            p95_compliant,
            p99_compliant,
            quality_compliant,
            resource_compliant,
            overall_compliant: p95_compliant && p99_compliant && quality_compliant && resource_compliant,
            violation_count: 0,
            critical_violations: vec![],
        })
    }
}

impl ViolationTracker {
    pub fn new() -> Self {
        Self {
            violations: Vec::new(),
            violation_counts: HashMap::new(),
        }
    }
}

impl BenchmarkResultsStorage {
    pub fn new() -> Self {
        Self {
            results: HashMap::new(),
            comparison_results: HashMap::new(),
            historical_data: VecDeque::new(),
        }
    }
    
    pub fn store_result(&mut self, result: BenchmarkResult) -> Result<()> {
        self.results.insert(result.benchmark_id.clone(), result);
        Ok(())
    }
    
    pub fn store_comparison(&mut self, comparison: ComparisonResult) -> Result<()> {
        let key = format!("{}_{}", comparison.baseline_id, comparison.optimized_id);
        self.comparison_results.insert(key, comparison);
        Ok(())
    }
}

// Default implementations for serializable types
impl Default for LatencyStatistics {
    fn default() -> Self {
        Self {
            p50_ms: 0.0,
            p95_ms: 0.0,
            p99_ms: 0.0,
            p999_ms: 0.0,
            mean_ms: 0.0,
            std_dev_ms: 0.0,
            min_ms: 0.0,
            max_ms: 0.0,
            sample_count: 0,
        }
    }
}

impl Default for ThroughputStatistics {
    fn default() -> Self {
        Self {
            peak_qps: 0.0,
            sustained_qps: 0.0,
            average_qps: 0.0,
            total_requests: 0,
            successful_requests: 0,
            failed_requests: 0,
            success_rate: 0.0,
        }
    }
}

impl Default for ResourceStatistics {
    fn default() -> Self {
        Self {
            peak_memory_mb: 0.0,
            average_memory_mb: 0.0,
            peak_cpu_percent: 0.0,
            average_cpu_percent: 0.0,
            memory_efficiency: 0.0,
            zero_copy_ratio: 0.0,
        }
    }
}

impl Default for QualityStatistics {
    fn default() -> Self {
        Self {
            average_recall_at_50: 0.0,
            average_precision: 0.0,
            average_f1_score: 0.0,
            quality_stability: 0.0,
            regression_detected: false,
        }
    }
}

impl Default for SlaComplianceReport {
    fn default() -> Self {
        Self {
            p95_compliant: false,
            p99_compliant: false,
            quality_compliant: false,
            resource_compliant: false,
            overall_compliant: false,
            violation_count: 0,
            critical_violations: vec![],
        }
    }
}

impl Default for OptimizationMetrics {
    fn default() -> Self {
        Self {
            latency_improvement_percent: 0.0,
            throughput_improvement_percent: 0.0,
            memory_savings_percent: 0.0,
            cpu_efficiency_improvement: 0.0,
            fusion_effectiveness: 0.0,
            parallel_efficiency: 0.0,
            early_stopping_savings: 0.0,
            prefetch_hit_rate: 0.0,
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[tokio::test]
    async fn test_benchmarker_creation() {
        let baseline_config = PipelineConfig::default();
        let optimized_config = PipelineConfig::default();
        let benchmark_config = BenchmarkConfig::default();
        
        let benchmarker = PipelineBenchmarker::new(
            baseline_config,
            optimized_config,
            benchmark_config,
        ).await;
        
        assert!(benchmarker.is_ok());
    }

    #[test]
    fn test_percentile_calculation() {
        let values = vec![10, 20, 30, 40, 50, 60, 70, 80, 90, 100];
        
        assert_eq!(PipelineBenchmarker::percentile(&values, 50.0), 50.0);
        assert_eq!(PipelineBenchmarker::percentile(&values, 95.0), 100.0);
        assert_eq!(PipelineBenchmarker::percentile(&values, 0.0), 10.0);
    }

    #[test]
    fn test_std_deviation() {
        let values = vec![10, 20, 30, 40, 50];
        let std_dev = PipelineBenchmarker::std_deviation(&values);
        
        // Standard deviation should be approximately 15.8
        assert!((std_dev - 15.8).abs() < 1.0);
    }

    #[tokio::test]
    async fn test_sla_validation() {
        let targets = SlaTargets {
            p95_latency_ms: 150,
            p99_latency_ms: 300,
            min_quality_score: 0.8,
            max_memory_mb: 512.0,
            max_cpu_percent: 80.0,
        };
        
        let validator = SlaValidator::new(targets);
        
        let result = BenchmarkResult {
            benchmark_id: "test".to_string(),
            timestamp: SystemTime::now(),
            configuration: "test".to_string(),
            test_suite: "test".to_string(),
            latency_stats: LatencyStatistics {
                p50_ms: 80.0,
                p95_ms: 140.0,
                p99_ms: 280.0,
                p999_ms: 350.0,
                mean_ms: 90.0,
                std_dev_ms: 25.0,
                min_ms: 50.0,
                max_ms: 400.0,
                sample_count: 1000,
            },
            throughput_stats: ThroughputStatistics {
                peak_qps: 100.0,
                average_qps: 80.0,
                sustained_qps: 75.0,
                ramp_up_time_s: 10.0,
                steady_state_duration_s: 60.0,
            },
            resource_stats: ResourceStatistics {
                peak_memory_mb: 400.0,
                average_memory_mb: 350.0,
                peak_cpu_percent: 70.0,
                average_cpu_percent: 55.0,
                memory_efficiency: 0.8,
                zero_copy_ratio: 0.9,
            },
            quality_stats: QualityStatistics {
                average_recall_at_50: 0.85,
                average_precision_at_10: 0.92,
                mrr: 0.88,
                ndcg_at_10: 0.90,
                expected_reciprocal_rank: 0.87,
                quality_trend: QualityTrend::Improving,
            },
            sla_compliance: SlaComplianceReport {
                p95_compliant: true,
                p99_compliant: true,
                quality_compliant: true,
                memory_compliant: true,
                overall_compliant: true,
                violations: Vec::new(),
            },
            optimization_metrics: OptimizationMetrics {
                latency_improvement_percent: 15.0,
                quality_improvement_percent: 8.0,
                memory_savings_percent: 12.0,
                optimization_confidence: 0.95,
            },
            execution_metadata: ExecutionMetadata {
                duration: Duration::from_secs(300),
                environment: "test".to_string(),
                version: "1.0.0".to_string(),
                git_commit: "abcd1234".to_string(),
            },
        };
        
        let compliance = validator.validate_result(&result).await.unwrap();
        assert!(compliance.overall_compliant);
    }

    #[test]
    fn test_benchmark_config_default() {
        let config = BenchmarkConfig::default();
        
        assert_eq!(config.warmup_duration, Duration::from_secs(5));
        assert_eq!(config.measurement_duration, Duration::from_secs(30));
        assert_eq!(config.concurrent_requests, 10);
        assert_eq!(config.iterations, 100);
        assert_eq!(config.percentiles, vec![50.0, 95.0, 99.0]);
    }

    #[test]
    fn test_sla_targets_creation() {
        let targets = SlaTargets {
            p95_latency_ms: 150,
            p99_latency_ms: 300,
            min_quality_score: 0.8,
            max_memory_mb: 512.0,
            max_cpu_percent: 80.0,
        };
        
        assert_eq!(targets.p95_latency_ms, 150);
        assert_eq!(targets.p99_latency_ms, 300);
        assert_eq!(targets.min_quality_score, 0.8);
        assert_eq!(targets.max_memory_mb, 512.0);
        assert_eq!(targets.max_cpu_percent, 80.0);
    }

    #[test]
    fn test_latency_statistics_default() {
        let stats = LatencyStatistics::default();
        
        assert_eq!(stats.mean_ms, 0.0);
        assert_eq!(stats.p50_ms, 0.0);
        assert_eq!(stats.p95_ms, 0.0);
        assert_eq!(stats.p99_ms, 0.0);
        assert_eq!(stats.min_ms, 0.0);
        assert_eq!(stats.max_ms, 0.0);
        assert_eq!(stats.std_dev_ms, 0.0);
        assert_eq!(stats.sample_count, 0);
    }

    #[test]
    fn test_throughput_statistics_default() {
        let stats = ThroughputStatistics::default();
        
        assert_eq!(stats.requests_per_second, 0.0);
        assert_eq!(stats.peak_rps, 0.0);
        assert_eq!(stats.total_requests, 0);
        assert_eq!(stats.successful_requests, 0);
        assert_eq!(stats.failed_requests, 0);
        assert_eq!(stats.error_rate, 0.0);
        assert_eq!(stats.timeout_rate, 0.0);
    }

    #[test]
    fn test_resource_statistics_default() {
        let stats = ResourceStatistics::default();
        
        assert_eq!(stats.peak_memory_mb, 0.0);
        assert_eq!(stats.average_memory_mb, 0.0);
        assert_eq!(stats.peak_cpu_percent, 0.0);
        assert_eq!(stats.average_cpu_percent, 0.0);
        assert_eq!(stats.disk_io_mb, 0.0);
        assert_eq!(stats.network_io_mb, 0.0);
    }

    #[test]
    fn test_quality_statistics_default() {
        let stats = QualityStatistics::default();
        
        assert_eq!(stats.average_recall_at_50, 0.0);
        assert_eq!(stats.average_precision_at_50, 0.0);
        assert_eq!(stats.average_f1_score, 0.0);
        assert_eq!(stats.ndcg_at_10, 0.0);
        assert_eq!(stats.map_at_50, 0.0);
        assert_eq!(stats.quality_distribution.len(), 0);
    }

    #[test]
    fn test_optimization_metrics_default() {
        let metrics = OptimizationMetrics::default();
        
        assert_eq!(metrics.fusion_effectiveness, 0.0);
        assert_eq!(metrics.parallel_efficiency, 0.0);
        assert_eq!(metrics.early_stopping_savings, 0.0);
        assert_eq!(metrics.prefetch_hit_rate, 0.0);
    }

    #[tokio::test]
    async fn test_sla_validator_creation() {
        let targets = SlaTargets {
            p95_latency_ms: 150,
            p99_latency_ms: 300,
            min_quality_score: 0.8,
            max_memory_mb: 512.0,
            max_cpu_percent: 80.0,
        };
        
        let validator = SlaValidator::new(targets);
        
        // Verify validator is created properly
        assert!(std::ptr::addr_of!(validator) != std::ptr::null());
    }

    #[tokio::test]
    async fn test_sla_validation_failure() {
        let targets = SlaTargets {
            p95_latency_ms: 150,
            p99_latency_ms: 300,
            min_quality_score: 0.8,
            max_memory_mb: 512.0,
            max_cpu_percent: 80.0,
        };
        
        let validator = SlaValidator::new(targets);
        
        let result = BenchmarkResult {
            benchmark_id: "test".to_string(),
            timestamp: SystemTime::now(),
            configuration: "test".to_string(),
            test_suite: "test".to_string(),
            latency_stats: LatencyStatistics {
                p95_ms: 200.0, // Exceeds target
                p99_ms: 350.0, // Exceeds target
                ..Default::default()
            },
            quality_stats: QualityStatistics {
                average_recall_at_50: 0.7, // Below target
                ..Default::default()
            },
            resource_stats: ResourceStatistics {
                peak_memory_mb: 600.0, // Exceeds target
                peak_cpu_percent: 90.0, // Exceeds target
                ..Default::default()
            },
            ..BenchmarkResult::default()
        };
        
        let compliance = validator.validate_result(&result).await.unwrap();
        assert!(!compliance.overall_compliant); // Should fail
    }

    #[tokio::test]
    async fn test_performance_measurement_system() {
        let config = PerformanceMeasurementConfig {
            enable_detailed_profiling: true,
            enable_memory_tracking: true,
            enable_cpu_tracking: true,
            sampling_interval: Duration::from_millis(100),
            max_sample_history: 1000,
        };
        
        let system = PerformanceMeasurementSystem::new(config);
        
        // Test measurement start/stop
        let session_id = system.start_measurement("test_session").await.unwrap();
        assert!(!session_id.is_empty());
        
        // Simulate some work
        sleep(Duration::from_millis(10)).await;
        
        let measurements = system.stop_measurement(&session_id).await.unwrap();
        
        // Verify measurements were collected
        assert!(measurements.duration > Duration::from_millis(0));
    }

    #[tokio::test]
    async fn test_statistics_engine_calculations() {
        let values = vec![10, 20, 30, 40, 50, 60, 70, 80, 90, 100];
        
        // Test different percentiles
        assert_eq!(PipelineBenchmarker::percentile(&values, 10.0), 10.0);
        assert_eq!(PipelineBenchmarker::percentile(&values, 25.0), 30.0);
        assert_eq!(PipelineBenchmarker::percentile(&values, 75.0), 80.0);
        assert_eq!(PipelineBenchmarker::percentile(&values, 90.0), 90.0);
        
        // Test edge cases
        let single_value = vec![42];
        assert_eq!(PipelineBenchmarker::percentile(&single_value, 50.0), 42.0);
        assert_eq!(PipelineBenchmarker::percentile(&single_value, 95.0), 42.0);
        
        let empty_values = vec![];
        assert_eq!(PipelineBenchmarker::percentile(&empty_values, 50.0), 0.0);
    }

    #[test]
    fn test_std_deviation_edge_cases() {
        // Single value
        let single = vec![42];
        assert_eq!(PipelineBenchmarker::std_deviation(&single), 0.0);
        
        // All same values
        let same_values = vec![10, 10, 10, 10, 10];
        assert_eq!(PipelineBenchmarker::std_deviation(&same_values), 0.0);
        
        // Empty values
        let empty = vec![];
        assert_eq!(PipelineBenchmarker::std_deviation(&empty), 0.0);
        
        // Two values
        let two_values = vec![10, 20];
        assert!((PipelineBenchmarker::std_deviation(&two_values) - 5.0).abs() < 0.1);
    }

    #[tokio::test]
    async fn test_benchmark_test_suite_creation() {
        let suite = BenchmarkTestSuite {
            name: "test_suite".to_string(),
            description: "Test suite description".to_string(),
            test_queries: vec![
                "test query 1".to_string(),
                "test query 2".to_string(),
                "test query 3".to_string(),
            ],
            expected_results: HashMap::new(),
            weight: 1.0,
            timeout: Duration::from_secs(5),
        };
        
        assert_eq!(suite.name, "test_suite");
        assert_eq!(suite.test_queries.len(), 3);
        assert_eq!(suite.weight, 1.0);
        assert_eq!(suite.timeout, Duration::from_secs(5));
    }

    #[test]
    fn test_benchmark_result_creation() {
        let result = BenchmarkResult {
            benchmark_id: "test_benchmark".to_string(),
            timestamp: SystemTime::now(),
            configuration: "baseline".to_string(),
            test_suite: "performance_suite".to_string(),
            latency_stats: LatencyStatistics {
                mean_ms: 120.0,
                median_ms: 115.0,
                p95_ms: 140.0,
                p99_ms: 180.0,
                min_ms: 80.0,
                max_ms: 200.0,
                std_dev_ms: 25.0,
            },
            throughput_stats: ThroughputStatistics {
                requests_per_second: 100.0,
                peak_rps: 150.0,
                total_requests: 1000,
                successful_requests: 980,
                failed_requests: 20,
                error_rate: 0.02,
                timeout_rate: 0.005,
            },
            resource_stats: ResourceStatistics {
                peak_memory_mb: 256.0,
                average_memory_mb: 200.0,
                peak_cpu_percent: 65.0,
                average_cpu_percent: 45.0,
                disk_io_mb: 10.5,
                network_io_mb: 5.2,
            },
            quality_stats: QualityStatistics {
                average_recall_at_50: 0.85,
                average_precision_at_50: 0.82,
                average_f1_score: 0.835,
                ndcg_at_10: 0.78,
                map_at_50: 0.80,
                quality_distribution: HashMap::new(),
            },
            sla_compliance: SlaComplianceReport::default(),
            optimization_metrics: OptimizationMetrics {
                fusion_effectiveness: 0.15,
                parallel_efficiency: 0.8,
                early_stopping_savings: 0.12,
                prefetch_hit_rate: 0.65,
            },
            execution_metadata: ExecutionMetadata {
                duration: Duration::from_secs(30),
                warmup_duration: Duration::from_secs(5),
                test_environment: "test".to_string(),
                git_commit: Some("abc123".to_string()),
                benchmark_version: "1.0.0".to_string(),
                additional_metadata: HashMap::new(),
            },
        };
        
        assert_eq!(result.benchmark_id, "test_benchmark");
        assert_eq!(result.configuration, "baseline");
        assert_eq!(result.latency_stats.mean_ms, 120.0);
        assert_eq!(result.throughput_stats.requests_per_second, 100.0);
        assert_eq!(result.resource_stats.peak_memory_mb, 256.0);
        assert_eq!(result.quality_stats.average_recall_at_50, 0.85);
        assert_eq!(result.optimization_metrics.fusion_effectiveness, 0.15);
    }

    #[tokio::test]
    async fn test_concurrent_benchmarks() {
        // Test that multiple benchmarks can run concurrently
        let baseline_config = PipelineConfig::default();
        let optimized_config = PipelineConfig::default();
        let benchmark_config = BenchmarkConfig::default();
        
        let benchmarker = PipelineBenchmarker::new(
            baseline_config,
            optimized_config,
            benchmark_config,
        ).await.unwrap();
        
        // Run multiple concurrent benchmark sessions
        let tasks: Vec<_> = (0..3).map(|i| {
            let suite_name = format!("concurrent_suite_{}", i);
            async move {
                // Simulate benchmark work
                sleep(Duration::from_millis(50)).await;
                suite_name
            }
        }).collect();
        
        let results = futures::future::join_all(tasks).await;
        assert_eq!(results.len(), 3);
        
        for (i, result) in results.into_iter().enumerate() {
            assert_eq!(result, format!("concurrent_suite_{}", i));
        }
    }

    #[test]
    fn test_sla_compliance_report_default() {
        let report = SlaComplianceReport::default();
        
        assert_eq!(report.overall_compliant, false);
        assert_eq!(report.latency_compliant, false);
        assert_eq!(report.resource_compliant, false);
        assert_eq!(report.quality_compliant, false);
        assert!(report.violations.is_empty());
        assert!(report.warnings.is_empty());
    }

    #[test]
    fn test_execution_metadata_creation() {
        let mut metadata = HashMap::new();
        metadata.insert("key1".to_string(), "value1".to_string());
        metadata.insert("key2".to_string(), "value2".to_string());
        
        let exec_metadata = ExecutionMetadata {
            duration: Duration::from_secs(45),
            warmup_duration: Duration::from_secs(10),
            test_environment: "production-like".to_string(),
            git_commit: Some("def456".to_string()),
            benchmark_version: "2.0.0".to_string(),
            additional_metadata: metadata.clone(),
        };
        
        assert_eq!(exec_metadata.duration, Duration::from_secs(45));
        assert_eq!(exec_metadata.warmup_duration, Duration::from_secs(10));
        assert_eq!(exec_metadata.test_environment, "production-like");
        assert_eq!(exec_metadata.git_commit, Some("def456".to_string()));
        assert_eq!(exec_metadata.benchmark_version, "2.0.0");
        assert_eq!(exec_metadata.additional_metadata.len(), 2);
        assert_eq!(exec_metadata.additional_metadata.get("key1"), Some(&"value1".to_string()));
    }

    #[test]
    fn test_latency_statistics_calculations() {
        let mut stats = LatencyStatistics::default();
        
        // Update with sample data
        stats.mean_ms = 105.0;
        stats.median_ms = 100.0;
        stats.p95_ms = 145.0;
        stats.p99_ms = 180.0;
        stats.min_ms = 75.0;
        stats.max_ms = 200.0;
        stats.std_dev_ms = 20.0;
        
        // Verify values
        assert_eq!(stats.mean_ms, 105.0);
        assert_eq!(stats.median_ms, 100.0);
        assert_eq!(stats.p95_ms, 145.0);
        assert_eq!(stats.p99_ms, 180.0);
        assert_eq!(stats.min_ms, 75.0);
        assert_eq!(stats.max_ms, 200.0);
        assert_eq!(stats.std_dev_ms, 20.0);
        
        // Verify p95 < p99
        assert!(stats.p95_ms < stats.p99_ms);
        assert!(stats.min_ms < stats.max_ms);
        assert!(stats.median_ms >= stats.min_ms);
        assert!(stats.median_ms <= stats.max_ms);
    }

    #[test]
    fn test_quality_distribution() {
        let mut quality_stats = QualityStatistics::default();
        
        // Add quality distribution data
        let mut distribution = HashMap::new();
        distribution.insert("excellent".to_string(), 25);
        distribution.insert("good".to_string(), 40);
        distribution.insert("fair".to_string(), 20);
        distribution.insert("poor".to_string(), 15);
        
        quality_stats.quality_distribution = distribution.clone();
        
        assert_eq!(quality_stats.quality_distribution.len(), 4);
        assert_eq!(quality_stats.quality_distribution.get("excellent"), Some(&25));
        assert_eq!(quality_stats.quality_distribution.get("good"), Some(&40));
        assert_eq!(quality_stats.quality_distribution.get("fair"), Some(&20));
        assert_eq!(quality_stats.quality_distribution.get("poor"), Some(&15));
        
        // Test total count
        let total: i32 = quality_stats.quality_distribution.values().sum();
        assert_eq!(total, 100);
    }

    #[test]
    fn test_benchmark_config_customization() {
        let custom_config = BenchmarkConfig {
            warmup_duration: Duration::from_secs(10),
            measurement_duration: Duration::from_secs(60),
            concurrent_requests: 20,
            iterations: 500,
            percentiles: vec![25.0, 50.0, 75.0, 90.0, 95.0, 99.0, 99.9],
            enable_detailed_metrics: true,
            enable_resource_monitoring: true,
            max_acceptable_error_rate: 0.05,
            target_rps: Some(150.0),
            memory_limit_mb: Some(1024.0),
            cpu_limit_percent: Some(85.0),
        };
        
        assert_eq!(custom_config.warmup_duration, Duration::from_secs(10));
        assert_eq!(custom_config.measurement_duration, Duration::from_secs(60));
        assert_eq!(custom_config.concurrent_requests, 20);
        assert_eq!(custom_config.iterations, 500);
        assert_eq!(custom_config.percentiles.len(), 7);
        assert_eq!(custom_config.enable_detailed_metrics, true);
        assert_eq!(custom_config.enable_resource_monitoring, true);
        assert_eq!(custom_config.max_acceptable_error_rate, 0.05);
        assert_eq!(custom_config.target_rps, Some(150.0));
        assert_eq!(custom_config.memory_limit_mb, Some(1024.0));
        assert_eq!(custom_config.cpu_limit_percent, Some(85.0));
    }

    #[test]
    fn test_error_rate_calculations() {
        let stats = ThroughputStatistics {
            requests_per_second: 95.0,
            peak_rps: 120.0,
            total_requests: 10000,
            successful_requests: 9800,
            failed_requests: 200,
            error_rate: 0.02, // 2%
            timeout_rate: 0.005, // 0.5%
        };
        
        // Verify error rate calculation
        let calculated_error_rate = stats.failed_requests as f64 / stats.total_requests as f64;
        assert!((calculated_error_rate - stats.error_rate).abs() < 0.001);
        
        // Verify success rate
        let success_rate = stats.successful_requests as f64 / stats.total_requests as f64;
        assert_eq!(success_rate, 0.98);
        
        // Verify totals
        assert_eq!(stats.successful_requests + stats.failed_requests, stats.total_requests);
    }
}

impl Default for BenchmarkResult {
    fn default() -> Self {
        Self {
            benchmark_id: String::new(),
            timestamp: UNIX_EPOCH,
            configuration: String::new(),
            test_suite: String::new(),
            latency_stats: LatencyStatistics::default(),
            throughput_stats: ThroughputStatistics::default(),
            resource_stats: ResourceStatistics::default(),
            quality_stats: QualityStatistics::default(),
            sla_compliance: SlaComplianceReport::default(),
            optimization_metrics: OptimizationMetrics::default(),
            execution_metadata: ExecutionMetadata {
                duration: Duration::from_secs(0),
                warmup_duration: Duration::from_secs(0),
                test_environment: String::new(),
                pipeline_version: String::new(),
                optimization_flags: vec![],
            },
        }
    }
}