siftdb_core/

compare.rs

use crate::bench::BenchmarkResults;
use serde::{Serialize, Deserialize};
use std::path::Path;
use std::fs;
use anyhow::Result;
use std::collections::HashMap;

#[derive(Serialize, Deserialize, Debug)]
pub struct BenchmarkComparison {
    pub baseline: BenchmarkResults,
    pub current: BenchmarkResults,
    pub performance_delta: PerformanceDelta,
}

#[derive(Serialize, Deserialize, Debug)]
pub struct PerformanceDelta {
    pub duration_change_percent: f64,
    pub throughput_change_percent: Option<f64>,
    pub queries_per_second_change_percent: Option<f64>,
    pub files_processed_change: i64,
    pub bytes_processed_change: i64,
}

#[derive(Serialize, Deserialize, Debug)]
pub struct RegressionReport {
    pub comparisons: Vec<BenchmarkComparison>,
    pub summary: RegressionSummary,
}

#[derive(Serialize, Deserialize, Debug)]
pub struct RegressionSummary {
    pub total_benchmarks: usize,
    pub regressions: usize,
    pub improvements: usize,
    pub avg_performance_change: f64,
}

pub struct BenchmarkAnalyzer;

impl BenchmarkAnalyzer {
    /// Compare two benchmark results and calculate performance delta
    pub fn compare_benchmarks(baseline: &BenchmarkResults, current: &BenchmarkResults) -> BenchmarkComparison {
        let duration_change = ((current.duration.as_secs_f64() - baseline.duration.as_secs_f64()) 
            / baseline.duration.as_secs_f64()) * 100.0;
        
        let throughput_change = match (baseline.throughput_mbps, current.throughput_mbps) {
            (Some(base), Some(curr)) => Some(((curr - base) / base) * 100.0),
            _ => None,
        };
        
        let qps_change = match (baseline.queries_per_second, current.queries_per_second) {
            (Some(base), Some(curr)) => Some(((curr - base) / base) * 100.0),
            _ => None,
        };
        
        BenchmarkComparison {
            baseline: baseline.clone(),
            current: current.clone(),
            performance_delta: PerformanceDelta {
                duration_change_percent: duration_change,
                throughput_change_percent: throughput_change,
                queries_per_second_change_percent: qps_change,
                files_processed_change: current.files_processed as i64 - baseline.files_processed as i64,
                bytes_processed_change: current.bytes_processed as i64 - baseline.bytes_processed as i64,
            },
        }
    }
    
    /// Load baseline benchmarks from directory
    pub fn load_baseline_benchmarks(benchmarks_dir: &Path) -> Result<HashMap<String, BenchmarkResults>> {
        let baseline_path = benchmarks_dir.join("baseline-0.1.json");
        if !baseline_path.exists() {
            anyhow::bail!("Baseline benchmark file not found: {:?}", baseline_path);
        }
        
        let content = fs::read_to_string(&baseline_path)?;
        let baseline: BenchmarkResults = serde_json::from_str(&content)?;
        
        let mut baselines = HashMap::new();
        baselines.insert(baseline.name.clone(), baseline);
        Ok(baselines)
    }
    
    /// Load current benchmark results from results directory
    pub fn load_current_benchmarks(results_dir: &Path) -> Result<Vec<BenchmarkResults>> {
        let mut results = Vec::new();
        
        if results_dir.exists() {
            for entry in fs::read_dir(results_dir)? {
                let entry = entry?;
                let path = entry.path();
                
                if path.extension().map_or(false, |ext| ext == "json") {
                    let content = fs::read_to_string(&path)?;
                    if let Ok(result) = serde_json::from_str::<BenchmarkResults>(&content) {
                        results.push(result);
                    }
                }
            }
        }
        
        Ok(results)
    }
    
    /// Generate regression report comparing baseline vs current benchmarks
    pub fn generate_regression_report(benchmarks_dir: &Path) -> Result<RegressionReport> {
        let baselines = Self::load_baseline_benchmarks(benchmarks_dir)?;
        let current_results = Self::load_current_benchmarks(&benchmarks_dir.join("results"))?;
        
        let mut comparisons = Vec::new();
        let mut total_change = 0.0;
        let mut regressions = 0;
        let mut improvements = 0;
        
        for current in &current_results {
            if let Some(baseline) = baselines.get(&current.name) {
                let comparison = Self::compare_benchmarks(baseline, current);
                
                // Count regressions/improvements based on queries per second
                if let Some(qps_change) = comparison.performance_delta.queries_per_second_change_percent {
                    total_change += qps_change;
                    if qps_change < -5.0 { // More than 5% slower is a regression
                        regressions += 1;
                    } else if qps_change > 5.0 { // More than 5% faster is an improvement
                        improvements += 1;
                    }
                }
                
                comparisons.push(comparison);
            }
        }
        
        let avg_change = if comparisons.is_empty() { 0.0 } else { total_change / comparisons.len() as f64 };
        
        Ok(RegressionReport {
            comparisons,
            summary: RegressionSummary {
                total_benchmarks: current_results.len(),
                regressions,
                improvements,
                avg_performance_change: avg_change,
            },
        })
    }
    
    /// Save regression report to file
    pub fn save_regression_report(report: &RegressionReport, output_path: &Path) -> Result<()> {
        let json_content = serde_json::to_string_pretty(report)?;
        fs::write(output_path, json_content)?;
        Ok(())
    }
    
    /// Print regression report summary to console
    pub fn print_regression_summary(report: &RegressionReport) {
        println!("🔍 SiftDB Performance Regression Report");
        println!("═══════════════════════════════════════");
        println!("Total Benchmarks: {}", report.summary.total_benchmarks);
        println!("Regressions (>5% slower): {}", report.summary.regressions);
        println!("Improvements (>5% faster): {}", report.summary.improvements);
        println!("Average Performance Change: {:.2}%", report.summary.avg_performance_change);
        println!();
        
        for comparison in &report.comparisons {
            println!("📊 Benchmark: {}", comparison.current.name);
            println!("   Duration: {:.2}% change", comparison.performance_delta.duration_change_percent);
            
            if let Some(qps_change) = comparison.performance_delta.queries_per_second_change_percent {
                let status = if qps_change < -5.0 {
                    "🔴 REGRESSION"
                } else if qps_change > 5.0 {
                    "🟢 IMPROVEMENT"
                } else {
                    "🟡 STABLE"
                };
                println!("   Queries/sec: {:.2}% change ({})", qps_change, status);
            }
            
            if let Some(throughput_change) = comparison.performance_delta.throughput_change_percent {
                println!("   Throughput: {:.2}% change", throughput_change);
            }
            println!();
        }
    }
}