lethe-core-rust 0.1.1

#!/usr/bin/env python3
"""
Lethe Regression Testing Framework
Post-Rust Refactor Validation Protocol

Implements the comprehensive regression testing protocol for validating
Rust refactor changes against frozen baselines with statistical rigor.
"""

import os
import sys
import json
import yaml
import subprocess
import tempfile
from pathlib import Path
from typing import Dict, List, Optional, Tuple
from dataclasses import dataclass, asdict
from datetime import datetime
import logging
import hashlib
import shutil

@dataclass
class ValidationResult:
    """Results from validation checks"""
    passed: bool
    metric: str
    expected: float
    actual: float
    threshold: float
    message: str

@dataclass
class RegressionResults:
    """Complete regression test results"""
    control_run_id: str
    test_run_id: str
    pool_fingerprint_match: bool
    hard_invariants: List[ValidationResult]
    soft_invariants: List[ValidationResult]
    calibration_results: Dict[str, float]
    performance_metrics: Dict[str, float]
    passed: bool

class RegressionTester:
    """Main regression testing orchestrator"""
    
    def __init__(self, config_path: str = "packages/benchmarking/benchmark_config.yaml"):
        self.config_path = Path(config_path)
        self.load_config()
        self.setup_logging()
        
        # Key paths
        self.results_dir = Path("benchmark_results")
        self.cache_dir = Path(".lethe_cache")
        self.pools_dir = Path("pools")
        
        # Validation thresholds (hard invariants)
        self.HARD_THRESHOLDS = {
            'budget_presence': 1.0,
            'paired_counts_equal': 1.0,
            'p95_vs_avg_ratio': 1.0,
            'p99_p95_ratio': 2.5,
            'causal_closure': 1.0,
            'proxy_gap': 0.005,  # 0.5%
            'ece_threshold': 0.08,
            'pool_fingerprint_match': 1.0
        }
        
        # Soft thresholds (performance expectations)
        self.SOFT_THRESHOLDS = {
            'latency_regression': 1.10,  # 10% regression allowed
            'recall_regression': 0.95,   # 5% recall drop allowed
            'kv_reuse_drop': 0.10,       # 10pp drop allowed
        }

    def load_config(self):
        """Load benchmark configuration"""
        try:
            with open(self.config_path, 'r') as f:
                self.config = yaml.safe_load(f)
        except FileNotFoundError:
            # Fallback minimal config
            self.config = {
                'run_name': 'regression_test',
                'random_seed': 42,
                'evaluation': {
                    'keep_ratios': [0.08, 0.15, 0.30],
                    'statistical_testing': {
                        'bootstrap_iterations': 100,  # Reduced for speed
                        'confidence_level': 0.95
                    }
                }
            }

    def setup_logging(self):
        """Setup logging for regression testing"""
        logging.basicConfig(
            level=logging.INFO,
            format='%(asctime)s - %(levelname)s - %(message)s',
            handlers=[
                logging.StreamHandler(sys.stdout),
                logging.FileHandler('regression_test.log')
            ]
        )
        self.logger = logging.getLogger(__name__)

    def clean_environment(self) -> bool:
        """Phase 0: Clean environment and restore ground truth"""
        self.logger.info("🧹 Cleaning environment and caches...")
        
        try:
            # Clear caches
            cache_dirs = [
                self.cache_dir,
                Path.home() / ".lethe" / "cache",
                Path.home() / ".lethe" / "indices"
            ]
            
            for cache_dir in cache_dirs:
                if cache_dir.exists():
                    shutil.rmtree(cache_dir)
                    self.logger.info(f"   Cleared cache: {cache_dir}")
            
            # Set deterministic mode
            os.environ['LETHE_DETERMINISTIC'] = '1'
            os.environ['LETHE_CACHE_DIR'] = str(self.cache_dir)
            
            self.logger.info("✅ Environment cleaned successfully")
            return True
            
        except Exception as e:
            self.logger.error(f"❌ Environment cleanup failed: {e}")
            return False

    def replay_control(self, matrix_path: str) -> Optional[str]:
        """Replay last known good configuration"""
        self.logger.info("🎮 Replaying control configuration...")
        
        try:
            # Find the most recent successful run
            if self.results_dir.exists():
                run_dirs = [d for d in self.results_dir.iterdir() if d.is_dir()]
                if run_dirs:
                    latest_run = max(run_dirs, key=lambda d: d.stat().st_mtime)
                    control_manifest = latest_run / "manifest.json"
                    
                    if control_manifest.exists():
                        self.logger.info(f"   Found control run: {latest_run.name}")
                        return latest_run.name
            
            # If no previous run, create baseline using actual benchmarking system
            lethe_root = Path(__file__).parent.parent.parent
            cmd = [
                "python3", "lethe-research/scripts/run_hybrid_infinitebench.py",
                "--mode", "full-evaluation",
                "--keep-ratios", "0.08,0.15,0.30",
                "--methods", "streaming,lethe,hybrid"
            ]
            
            result = subprocess.run(cmd, capture_output=True, text=True, timeout=3600, cwd=str(lethe_root))
            if result.returncode == 0:
                # Extract run ID from output
                lines = result.stdout.split('\n')
                for line in lines:
                    if "Run ID:" in line:
                        run_id = line.split("Run ID:")[-1].strip()
                        self.logger.info(f"✅ Control run completed: {run_id}")
                        return run_id
            
            self.logger.error(f"❌ Control run failed: {result.stderr}")
            return None
            
        except Exception as e:
            self.logger.error(f"❌ Control replay failed: {e}")
            return None

    def run_smoke_test(self) -> Tuple[bool, Optional[str]]:
        """Phase 1: Run 3x3 smoke test matrix"""
        self.logger.info("💨 Running smoke test (3x3 mini-matrix)...")
        
        try:
            # Create mini matrix configuration
            mini_config = {
                'datasets': ['infinitebench_sample'],
                'budgets': [0.08, 0.15, 0.30],
                'k_values': [1, 5, 10],
                'seeds': [42],
                'competitors': ['lethe_hybrid', 'weaviate_baseline']
            }
            
            with tempfile.NamedTemporaryFile(mode='w', suffix='.yml', delete=False) as f:
                yaml.dump(mini_config, f)
                mini_matrix_path = f.name
            
            # Use the actual benchmarking system for smoke test
            lethe_root = Path(__file__).parent.parent.parent
            cmd = [
                "python3", "lethe-research/scripts/run_hybrid_infinitebench.py",
                "--mode", "quick-test",
                "--keep-ratios", "0.08,0.15,0.30",
                "--methods", "streaming,lethe,hybrid"
            ]
            
            result = subprocess.run(cmd, capture_output=True, text=True, timeout=1800, cwd=str(lethe_root))  # 30min timeout
            
            # Cleanup temp file
            os.unlink(mini_matrix_path)
            
            if result.returncode == 0:
                # Extract run ID
                lines = result.stdout.split('\n')
                for line in lines:
                    if "Run ID:" in line:
                        run_id = line.split("Run ID:")[-1].strip()
                        self.logger.info(f"✅ Smoke test passed: {run_id}")
                        return True, run_id
            
            self.logger.error(f"❌ Smoke test failed: {result.stderr}")
            return False, None
            
        except subprocess.TimeoutExpired:
            self.logger.error("❌ Smoke test timed out")
            return False, None
        except Exception as e:
            self.logger.error(f"❌ Smoke test error: {e}")
            return False, None

    def validate_hard_invariants(self, run_id: str, control_run_id: str = None) -> List[ValidationResult]:
        """Validate hard invariants that must never fail"""
        self.logger.info("🔒 Validating hard invariants...")
        
        invariants = []
        results_file = self.results_dir / run_id / "results.json"
        
        if not results_file.exists():
            return [ValidationResult(False, "results_file", 1.0, 0.0, 1.0, "Results file not found")]
        
        try:
            with open(results_file, 'r') as f:
                results = json.load(f)
            
            # 1. Budget presence check
            budgets_present = len(results.get('budgets', [])) > 0
            invariants.append(ValidationResult(
                budgets_present, "budget_presence", 1.0, float(budgets_present), 1.0,
                "Budget constraints must be present"
            ))
            
            # 2. Paired counts equal
            paired_equal = results.get('paired_counts_match', False)
            invariants.append(ValidationResult(
                paired_equal, "paired_counts", 1.0, float(paired_equal), 1.0,
                "Paired comparison counts must be equal"
            ))
            
            # 3. P95 >= Average latency
            p95 = results.get('latency_p95', 0)
            avg = results.get('latency_avg', 0)
            p95_valid = p95 >= avg if avg > 0 else True
            invariants.append(ValidationResult(
                p95_valid, "p95_vs_avg", avg, p95, avg,
                "P95 latency must be >= average latency"
            ))
            
            # 4. P99/P95 ratio <= 2.5
            p99 = results.get('latency_p99', 0)
            ratio = p99 / p95 if p95 > 0 else 1.0
            ratio_valid = ratio <= 2.5
            invariants.append(ValidationResult(
                ratio_valid, "p99_p95_ratio", 2.5, ratio, 2.5,
                "P99/P95 ratio must be <= 2.5"
            ))
            
            # 5. Causal closure = 1.0
            closure = results.get('causal_closure', 0.0)
            closure_valid = abs(closure - 1.0) < 0.001
            invariants.append(ValidationResult(
                closure_valid, "causal_closure", 1.0, closure, 0.001,
                "Causal closure must be 1.0"
            ))
            
            # 6. Proxy gap <= 0.5%
            proxy_gap = results.get('proxy_gap', 1.0)
            gap_valid = proxy_gap <= 0.005
            invariants.append(ValidationResult(
                gap_valid, "proxy_gap", 0.005, proxy_gap, 0.005,
                "Proxy gap must be <= 0.5%"
            ))
            
            # 7. ECE <= 0.08
            ece = results.get('expected_calibration_error', 1.0)
            ece_valid = ece <= 0.08
            invariants.append(ValidationResult(
                ece_valid, "ece", 0.08, ece, 0.08,
                "Expected Calibration Error must be <= 0.08"
            ))
            
            # 8. Pool fingerprint match (if control provided)
            if control_run_id:
                control_file = self.results_dir / control_run_id / "results.json"
                if control_file.exists():
                    with open(control_file, 'r') as f:
                        control_results = json.load(f)
                    
                    test_fp = results.get('pool_fingerprint', '')
                    control_fp = control_results.get('pool_fingerprint', '')
                    fp_match = test_fp == control_fp
                    invariants.append(ValidationResult(
                        fp_match, "pool_fingerprint", 1.0, float(fp_match), 1.0,
                        f"Pool fingerprint must match control: {control_fp[:8]}..."
                    ))
            
            self.logger.info(f"   Validated {len(invariants)} hard invariants")
            return invariants
            
        except Exception as e:
            self.logger.error(f"❌ Hard invariant validation failed: {e}")
            return [ValidationResult(False, "validation_error", 1.0, 0.0, 1.0, str(e))]

    def diagnose_failures(self, failed_invariants: List[ValidationResult]) -> Dict[str, str]:
        """Diagnose specific failure patterns and suggest fixes"""
        self.logger.info("🔍 Diagnosing failures and suggesting fixes...")
        
        fixes = {}
        
        for invariant in failed_invariants:
            if not invariant.passed:
                if invariant.metric == "pool_fingerprint":
                    fixes[invariant.metric] = """
FROZEN POOL MISMATCH - Fix Path A:
1. Lock to signed manifest: --pool-manifest manifest.json --pool-mode frozen_only
2. If tokenization changed, regenerate frozen union pool and re-baseline
3. Quarantine mismatched competitors as 'Not Comparable'
"""
                
                elif invariant.metric in ["ece", "causal_closure"]:
                    fixes[invariant.metric] = """
CALIBRATION/ECE VIOLATION - Fix Path B:
1. Refit isotonic per type: lethe-calibrate fit --log logged_policy.jsonl --per-type --ips
2. Reduce σ-weight by ~20% and cap per-type variance shrinkage
3. Re-run mini-matrix with updated calibration
"""
                
                elif invariant.metric in ["proxy_gap", "p99_p95_ratio"]:
                    fixes[invariant.metric] = """  
OPTIMIZER MISBEHAVIOR - Fix Path C:
1. Enforce deterministic ordering: stable sort on (gain/token, hash)
2. Disable group-split temporarily: --group-split=off
3. Increase lazy cache freshness (recompute marginals every N=8)
4. Verify 64-bit hash stability across OS/arch
"""
                
                elif "latency" in invariant.metric:
                    fixes[invariant.metric] = """
KV/LATENCY REGRESSION - Fix Path E:
1. Guard the head: keep head_keep≈0.12, move summaries earlier
2. Shrink tail stride/windows: s→2k, W→4k
3. Confirm CE early-exit enabled and sinks=96
4. Check prefix-Jaccard hasn't dropped >10pp
"""
        
        return fixes

    def run_full_regression_test(self, matrix_path: str = "packages/benchmarking/matrix.yml") -> RegressionResults:
        """Execute complete regression testing protocol"""
        self.logger.info("🚀 Starting full regression testing protocol...")
        
        # Phase 0: Clean environment
        if not self.clean_environment():
            return RegressionResults("", "", False, [], [], {}, {}, False)
        
        # Phase 0.1: Replay control
        control_run_id = self.replay_control(matrix_path)
        if not control_run_id:
            self.logger.warning("⚠️  No control run available, proceeding without baseline")
        
        # Phase 1: Smoke test
        smoke_passed, test_run_id = self.run_smoke_test()
        if not smoke_passed:
            self.logger.error("❌ Smoke test failed - aborting full regression")
            return RegressionResults(control_run_id or "", "", False, [], [], {}, {}, False)
        
        # Phase 2: Validate invariants
        hard_invariants = self.validate_hard_invariants(test_run_id, control_run_id)
        
        failed_hard = [inv for inv in hard_invariants if not inv.passed]
        if failed_hard:
            self.logger.error(f"❌ {len(failed_hard)} hard invariants failed")
            fixes = self.diagnose_failures(failed_hard)
            for metric, fix in fixes.items():
                self.logger.info(f"\n🔧 Fix for {metric}:{fix}")
            
            return RegressionResults(
                control_run_id or "", test_run_id, False, 
                hard_invariants, [], {}, {}, False
            )
        
        self.logger.info("✅ All hard invariants passed!")
        
        # Phase 3: If hard invariants pass, run full matrix
        if self.should_run_full_matrix():
            full_run_id = self.run_full_matrix(matrix_path)
            if full_run_id:
                test_run_id = full_run_id
        
        # Phase 4: Validate soft thresholds and performance
        soft_invariants = self.validate_soft_invariants(test_run_id, control_run_id)
        
        return RegressionResults(
            control_run_id or "", test_run_id, True,
            hard_invariants, soft_invariants, {}, {}, True
        )

    def should_run_full_matrix(self) -> bool:
        """Determine if we should proceed to full matrix"""
        # For now, always proceed if hard invariants pass
        return True

    def run_full_matrix(self, matrix_path: str) -> Optional[str]:
        """Run the complete benchmark matrix"""
        self.logger.info("📊 Running full benchmark matrix...")
        
        try:
            cmd = [
                "python", "packages/benchmarking/benchmarks/__main__.py",
                "run",
                "--matrix", matrix_path,
                "--paired", 
                "--frozen-pool",
                "--strict",
                "--resume"
            ]
            
            result = subprocess.run(cmd, capture_output=True, text=True, timeout=21600)  # 6h timeout
            
            if result.returncode == 0:
                lines = result.stdout.split('\n')
                for line in lines:
                    if "Run ID:" in line:
                        run_id = line.split("Run ID:")[-1].strip()
                        self.logger.info(f"✅ Full matrix completed: {run_id}")
                        return run_id
            
            self.logger.error(f"❌ Full matrix failed: {result.stderr}")
            return None
            
        except Exception as e:
            self.logger.error(f"❌ Full matrix error: {e}")
            return None

    def validate_soft_invariants(self, run_id: str, control_run_id: str = None) -> List[ValidationResult]:
        """Validate soft performance thresholds"""
        self.logger.info("📈 Validating performance thresholds...")
        
        # Placeholder - would implement actual performance comparison
        return []

    def generate_report(self, results: RegressionResults) -> str:
        """Generate regression testing report"""
        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
        report_file = f"regression_report_{timestamp}.html"
        
        html_content = f"""
<!DOCTYPE html>
<html>
<head>
    <title>Lethe Regression Test Report</title>
    <style>
        body {{ font-family: Arial, sans-serif; margin: 20px; }}
        .passed {{ color: green; }}
        .failed {{ color: red; }}
        .metric {{ margin: 10px 0; padding: 10px; border: 1px solid #ccc; }}
    </style>
</head>
<body>
    <h1>Lethe Regression Test Report</h1>
    <p><strong>Generated:</strong> {datetime.now()}</p>
    <p><strong>Overall Result:</strong> <span class="{'passed' if results.passed else 'failed'}">
        {'✅ PASSED' if results.passed else '❌ FAILED'}
    </span></p>
    
    <h2>Test Runs</h2>
    <ul>
        <li><strong>Control:</strong> {results.control_run_id}</li>
        <li><strong>Test:</strong> {results.test_run_id}</li>
        <li><strong>Pool Match:</strong> {'✅' if results.pool_fingerprint_match else '❌'}</li>
    </ul>
    
    <h2>Hard Invariants ({len(results.hard_invariants)})</h2>
"""
        
        for inv in results.hard_invariants:
            status = "passed" if inv.passed else "failed"
            icon = "✅" if inv.passed else "❌"
            html_content += f"""
    <div class="metric {status}">
        <strong>{icon} {inv.metric}</strong><br>
        Expected: {inv.expected}, Actual: {inv.actual}, Threshold: {inv.threshold}<br>
        {inv.message}
    </div>
"""
        
        html_content += "</body></html>"
        
        with open(report_file, 'w') as f:
            f.write(html_content)
        
        self.logger.info(f"📄 Report generated: {report_file}")
        return report_file


def main():
    """Main entry point for regression testing"""
    import argparse
    
    parser = argparse.ArgumentParser(description="Lethe Regression Testing Framework")
    parser.add_argument("--matrix", default="packages/benchmarking/matrix.yml", 
                       help="Matrix configuration file")
    parser.add_argument("--config", default="packages/benchmarking/benchmark_config.yaml",
                       help="Benchmark configuration file")
    parser.add_argument("--smoke-only", action="store_true",
                       help="Run only smoke test")
    parser.add_argument("--clean-only", action="store_true", 
                       help="Only clean environment")
    
    args = parser.parse_args()
    
    # Initialize tester
    tester = RegressionTester(args.config)
    
    if args.clean_only:
        success = tester.clean_environment()
        sys.exit(0 if success else 1)
    
    if args.smoke_only:
        success, run_id = tester.run_smoke_test()
        if success:
            tester.validate_hard_invariants(run_id)
        sys.exit(0 if success else 1)
    
    # Run full regression test
    results = tester.run_full_regression_test(args.matrix)
    
    # Generate report
    report_file = tester.generate_report(results)
    
    # Exit with appropriate code
    sys.exit(0 if results.passed else 1)


if __name__ == "__main__":
    main()