lens-core 1.0.0

// CALIB_V22 Production Activation System - D0 24-Hour Canary Controller
// Phase 1: Tight production-grade 4-gate canary with ladder progression

use std::sync::Arc;
use std::time::{Duration, SystemTime};
use std::collections::HashMap;
use tokio::time::{interval, sleep};
use tracing::{info, warn, error, debug};
use serde::{Serialize, Deserialize};
use thiserror::Error;

use crate::calibration::{
    manifest::{CalibrationManifest, ParityReport},
    fingerprint_publisher::{FingerprintPublisher, FingerPrint},
    sla_monitoring::SlaMonitor,
};

/// Production Activation Controller for CALIB_V22 D0 24-Hour Canary
/// Implements 4-gate ladder progression: 5%→25%→50%→100%
pub struct ProductionActivationController {
    /// Current canary configuration
    config: CanaryConfig,
    
    /// Current deployment state
    state: Arc<tokio::sync::RwLock<CanaryState>>,
    
    /// Gate enforcement system
    gate_enforcer: CanaryGateEnforcer,
    
    /// Green fingerprint publisher
    fingerprint_publisher: FingerprintPublisher,
    
    /// SLA monitoring for gate validation
    sla_monitor: Arc<SlaMonitor>,
    
    /// Smoke probe executor
    smoke_prober: SmokeProbeExecutor,
}

#[derive(Debug, Clone)]
pub struct CanaryConfig {
    /// Ladder progression stages: traffic percentage per repo bucket
    pub traffic_progression: Vec<TrafficStage>,
    
    /// Gate enforcement thresholds
    pub gate_thresholds: GateThresholds,
    
    /// Monitoring windows and intervals
    pub monitoring_config: MonitoringConfig,
    
    /// Auto-revert configuration
    pub revert_config: RevertConfig,
}

#[derive(Debug, Clone)]
pub struct TrafficStage {
    /// Traffic percentage for this stage (5%, 25%, 50%, 100%)
    pub traffic_percentage: f64,
    
    /// Duration to run at this stage before progression
    pub duration: Duration,
    
    /// Repo bucket selection strategy
    pub bucket_strategy: BucketStrategy,
}

#[derive(Debug, Clone)]
pub enum BucketStrategy {
    /// Select repos by hash modulo
    HashModulo { modulo: u64, remainder: u64 },
    /// Explicit repo list
    ExplicitRepos(Vec<String>),
    /// Randomized selection
    RandomizedSample { seed: u64 },
}

#[derive(Debug, Clone)]
pub struct GateThresholds {
    /// P99 latency threshold: <1ms
    pub p99_latency_ms: f64,
    
    /// AECE-τ threshold: ≤0.01 per slice
    pub aece_tau_threshold: f64,
    
    /// Median confidence shift threshold: ≤0.02
    pub confidence_shift_threshold: f64,
    
    /// SLA-Recall@50 delta threshold: =0
    pub sla_recall_delta_threshold: f64,
    
    /// Consecutive breach tolerance: 2 × 15min
    pub consecutive_breach_tolerance: u32,
    
    /// Breach window duration
    pub breach_window: Duration,
}

#[derive(Debug, Clone)]
pub struct MonitoringConfig {
    /// Gate evaluation interval
    pub gate_evaluation_interval: Duration,
    
    /// Smoke probe interval
    pub smoke_probe_interval: Duration,
    
    /// Metrics collection window
    pub metrics_window: Duration,
    
    /// Alert cooldown period
    pub alert_cooldown: Duration,
}

#[derive(Debug, Clone)]
pub struct RevertConfig {
    /// Enable automatic revert on gate failures
    pub auto_revert_enabled: bool,
    
    /// Revert execution timeout
    pub revert_timeout: Duration,
    
    /// Post-revert validation duration
    pub post_revert_validation: Duration,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CanaryState {
    /// Current stage in the progression
    pub current_stage: u32,
    
    /// Stage start time
    pub stage_start_time: SystemTime,
    
    /// Overall deployment start time
    pub deployment_start_time: SystemTime,
    
    /// Current gate status
    pub gate_status: GateStatus,
    
    /// Consecutive breach count
    pub consecutive_breaches: u32,
    
    /// Last gate evaluation time
    pub last_gate_evaluation: SystemTime,
    
    /// Canary deployment status
    pub deployment_status: CanaryDeploymentStatus,
    
    /// Error history
    pub error_history: Vec<CanaryError>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GateStatus {
    /// P99 latency gate status
    pub p99_latency: GateResult,
    
    /// AECE-τ gate status per slice
    pub aece_tau_gates: HashMap<String, GateResult>,
    
    /// Confidence shift gate status
    pub confidence_shift: GateResult,
    
    /// SLA-Recall@50 delta gate status
    pub sla_recall_delta: GateResult,
    
    /// Overall gate pass/fail
    pub overall_passed: bool,
    
    /// Last evaluation timestamp
    pub evaluation_timestamp: SystemTime,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GateResult {
    /// Gate passed/failed
    pub passed: bool,
    
    /// Measured value
    pub measured_value: f64,
    
    /// Threshold value
    pub threshold_value: f64,
    
    /// Measurement timestamp
    pub timestamp: SystemTime,
    
    /// Additional context
    pub context: Option<String>,
}

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub enum CanaryDeploymentStatus {
    /// Canary not started
    NotStarted,
    
    /// Currently running at specific stage
    Running { stage: u32, progress_percent: f64 },
    
    /// Progressing to next stage
    Progressing { from_stage: u32, to_stage: u32 },
    
    /// Successfully completed all stages
    Completed,
    
    /// Failed and reverted
    Failed { reason: String, reverted: bool },
    
    /// Currently reverting
    Reverting,
}

/// 4-Gate Canary Controller with Ladder Progression
pub struct CanaryGateEnforcer {
    thresholds: GateThresholds,
    monitoring_config: MonitoringConfig,
    sla_monitor: Arc<SlaMonitor>,
}

impl CanaryGateEnforcer {
    pub fn new(
        thresholds: GateThresholds,
        monitoring_config: MonitoringConfig,
        sla_monitor: Arc<SlaMonitor>,
    ) -> Self {
        Self {
            thresholds,
            monitoring_config,
            sla_monitor,
        }
    }
    
    /// Evaluate all gates for current canary stage
    pub async fn evaluate_gates(&self, stage: u32) -> Result<GateStatus, ActivationError> {
        debug!("🔍 Evaluating canary gates for stage {}", stage);
        
        let evaluation_start = SystemTime::now();
        
        // Gate 1: P99 Latency < 1ms
        let p99_latency = self.evaluate_p99_latency_gate().await?;
        
        // Gate 2: AECE-τ ≤ 0.01 per slice
        let aece_tau_gates = self.evaluate_aece_tau_gates().await?;
        
        // Gate 3: Median confidence shift ≤ 0.02
        let confidence_shift = self.evaluate_confidence_shift_gate().await?;
        
        // Gate 4: Δ(SLA-Recall@50) = 0
        let sla_recall_delta = self.evaluate_sla_recall_delta_gate().await?;
        
        // Overall gate evaluation
        let aece_tau_passed = aece_tau_gates.values().all(|g| g.passed);
        let overall_passed = p99_latency.passed 
            && aece_tau_passed 
            && confidence_shift.passed 
            && sla_recall_delta.passed;
        
        let gate_status = GateStatus {
            p99_latency,
            aece_tau_gates,
            confidence_shift,
            sla_recall_delta,
            overall_passed,
            evaluation_timestamp: evaluation_start,
        };
        
        if overall_passed {
            info!("✅ All canary gates passed for stage {}", stage);
        } else {
            warn!("❌ Canary gate failures detected for stage {}", stage);
        }
        
        Ok(gate_status)
    }
    
    async fn evaluate_p99_latency_gate(&self) -> Result<GateResult, ActivationError> {
        // Query actual P99 latency from monitoring system
        let p99_latency = self.sla_monitor.get_p99_latency_ms().await
            .map_err(|e| ActivationError::MetricsError(format!("P99 latency query failed: {}", e)))?;
        
        let passed = p99_latency <= self.thresholds.p99_latency_ms;
        
        Ok(GateResult {
            passed,
            measured_value: p99_latency,
            threshold_value: self.thresholds.p99_latency_ms,
            timestamp: SystemTime::now(),
            context: Some(format!("Current: {:.3}ms, Threshold: {:.3}ms", 
                p99_latency, self.thresholds.p99_latency_ms)),
        })
    }
    
    async fn evaluate_aece_tau_gates(&self) -> Result<HashMap<String, GateResult>, ActivationError> {
        let mut aece_tau_gates = HashMap::new();
        
        // Get AECE-τ per slice from monitoring
        let slice_metrics = self.sla_monitor.get_aece_tau_per_slice().await
            .map_err(|e| ActivationError::MetricsError(format!("AECE-τ query failed: {}", e)))?;
        
        for (slice_name, aece_tau_value) in slice_metrics {
            let passed = aece_tau_value <= self.thresholds.aece_tau_threshold;
            
            aece_tau_gates.insert(slice_name.clone(), GateResult {
                passed,
                measured_value: aece_tau_value,
                threshold_value: self.thresholds.aece_tau_threshold,
                timestamp: SystemTime::now(),
                context: Some(format!("Slice: {}, AECE-τ: {:.4}, Threshold: {:.4}", 
                    slice_name, aece_tau_value, self.thresholds.aece_tau_threshold)),
            });
        }
        
        Ok(aece_tau_gates)
    }
    
    async fn evaluate_confidence_shift_gate(&self) -> Result<GateResult, ActivationError> {
        // Query median confidence shift from monitoring
        let confidence_shift = self.sla_monitor.get_median_confidence_shift().await
            .map_err(|e| ActivationError::MetricsError(format!("Confidence shift query failed: {}", e)))?;
        
        let passed = confidence_shift.abs() <= self.thresholds.confidence_shift_threshold;
        
        Ok(GateResult {
            passed,
            measured_value: confidence_shift,
            threshold_value: self.thresholds.confidence_shift_threshold,
            timestamp: SystemTime::now(),
            context: Some(format!("Shift: {:.4}, Threshold: ±{:.4}", 
                confidence_shift, self.thresholds.confidence_shift_threshold)),
        })
    }
    
    async fn evaluate_sla_recall_delta_gate(&self) -> Result<GateResult, ActivationError> {
        // Query SLA-Recall@50 delta from monitoring
        let sla_recall_delta = self.sla_monitor.get_sla_recall_at_50_delta().await
            .map_err(|e| ActivationError::MetricsError(format!("SLA-Recall@50 delta query failed: {}", e)))?;
        
        let passed = sla_recall_delta.abs() <= self.thresholds.sla_recall_delta_threshold;
        
        Ok(GateResult {
            passed,
            measured_value: sla_recall_delta,
            threshold_value: self.thresholds.sla_recall_delta_threshold,
            timestamp: SystemTime::now(),
            context: Some(format!("Delta: {:.4}, Threshold: ±{:.4}", 
                sla_recall_delta, self.thresholds.sla_recall_delta_threshold)),
        })
    }
}

/// Smoke Probe Executor for Canary Validation
pub struct SmokeProbeExecutor {
    probe_config: SmokeProbeConfig,
}

#[derive(Debug, Clone)]
pub struct SmokeProbeConfig {
    /// Probe execution interval
    pub execution_interval: Duration,
    
    /// Probe timeout per execution
    pub probe_timeout: Duration,
    
    /// Number of probes per execution
    pub probes_per_execution: u32,
    
    /// Probe failure threshold
    pub failure_threshold: f64,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SmokeProbeResult {
    /// Probe execution timestamp
    pub timestamp: SystemTime,
    
    /// Identity-calibrated slice probe result
    pub identity_calibrated: ProbeResult,
    
    /// Discrete-plateau slice probe result
    pub discrete_plateau: ProbeResult,
    
    /// Skewed weights probe result
    pub skewed_weights: ProbeResult,
    
    /// Heavy-tail logits probe result
    pub heavy_tail_logits: ProbeResult,
    
    /// Overall probe success rate
    pub overall_success_rate: f64,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ProbeResult {
    /// Probe passed/failed
    pub passed: bool,
    
    /// Execution latency
    pub latency_ms: f64,
    
    /// Response correctness
    pub correctness_score: f64,
    
    /// Error message if failed
    pub error: Option<String>,
}

impl SmokeProbeExecutor {
    pub fn new(probe_config: SmokeProbeConfig) -> Self {
        Self { probe_config }
    }
    
    /// Execute smoke probes for canary validation
    pub async fn execute_smoke_probes(&self) -> Result<SmokeProbeResult, ActivationError> {
        debug!("🔬 Executing smoke probes for canary validation");
        
        let execution_start = SystemTime::now();
        
        // Execute all probe types in parallel
        let (identity_result, plateau_result, skewed_result, heavy_tail_result) = tokio::join!(
            self.execute_identity_calibrated_probe(),
            self.execute_discrete_plateau_probe(),
            self.execute_skewed_weights_probe(),
            self.execute_heavy_tail_logits_probe()
        );
        
        let identity_calibrated = identity_result?;
        let discrete_plateau = plateau_result?;
        let skewed_weights = skewed_result?;
        let heavy_tail_logits = heavy_tail_result?;
        
        // Calculate overall success rate
        let successful_probes = [
            &identity_calibrated,
            &discrete_plateau,
            &skewed_weights,
            &heavy_tail_logits,
        ].iter().filter(|p| p.passed).count();
        
        let overall_success_rate = successful_probes as f64 / 4.0;
        
        let probe_result = SmokeProbeResult {
            timestamp: execution_start,
            identity_calibrated,
            discrete_plateau,
            skewed_weights,
            heavy_tail_logits,
            overall_success_rate,
        };
        
        info!("🎯 Smoke probe execution completed with {:.1}% success rate", 
              overall_success_rate * 100.0);
        
        Ok(probe_result)
    }
    
    async fn execute_identity_calibrated_probe(&self) -> Result<ProbeResult, ActivationError> {
        let start_time = SystemTime::now();
        
        // Simulate identity-calibrated slice probe
        // In production, this would test actual calibrated identity function
        sleep(Duration::from_millis(5)).await;
        
        let latency_ms = start_time.elapsed().unwrap().as_secs_f64() * 1000.0;
        
        Ok(ProbeResult {
            passed: true,
            latency_ms,
            correctness_score: 0.98,
            error: None,
        })
    }
    
    async fn execute_discrete_plateau_probe(&self) -> Result<ProbeResult, ActivationError> {
        let start_time = SystemTime::now();
        
        // Simulate discrete-plateau slice probe
        sleep(Duration::from_millis(8)).await;
        
        let latency_ms = start_time.elapsed().unwrap().as_secs_f64() * 1000.0;
        
        Ok(ProbeResult {
            passed: true,
            latency_ms,
            correctness_score: 0.95,
            error: None,
        })
    }
    
    async fn execute_skewed_weights_probe(&self) -> Result<ProbeResult, ActivationError> {
        let start_time = SystemTime::now();
        
        // Simulate skewed weights probe
        sleep(Duration::from_millis(6)).await;
        
        let latency_ms = start_time.elapsed().unwrap().as_secs_f64() * 1000.0;
        
        Ok(ProbeResult {
            passed: true,
            latency_ms,
            correctness_score: 0.97,
            error: None,
        })
    }
    
    async fn execute_heavy_tail_logits_probe(&self) -> Result<ProbeResult, ActivationError> {
        let start_time = SystemTime::now();
        
        // Simulate heavy-tail logits probe
        sleep(Duration::from_millis(12)).await;
        
        let latency_ms = start_time.elapsed().unwrap().as_secs_f64() * 1000.0;
        
        Ok(ProbeResult {
            passed: true,
            latency_ms,
            correctness_score: 0.93,
            error: None,
        })
    }
}

/// Green Fingerprint Publisher for Production Attestation
pub struct GreenFingerprintPublisher {
    fingerprint_publisher: FingerprintPublisher,
    attestation_config: AttestationConfig,
}

#[derive(Debug, Clone)]
pub struct AttestationConfig {
    /// Cryptographic signing key
    pub signing_key: String,
    
    /// Attestation schema version
    pub schema_version: String,
    
    /// Required attestation fields
    pub required_fields: Vec<String>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ProductionFingerprint {
    /// Calibration manifest data
    pub calibration_manifest: CalibrationManifestData,
    
    /// Parity report data
    pub parity_report: ParityReportData,
    
    /// Weekly drift pack data
    pub weekly_drift_pack: WeeklyDriftPackData,
    
    /// Release fingerprint binding
    pub release_binding: ReleaseBinding,
    
    /// Cryptographic attestation
    pub attestation: CryptographicAttestation,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CalibrationManifestData {
    /// Calibration coefficients ĉ
    pub coefficients: Vec<f64>,
    
    /// Epsilon value ε
    pub epsilon: f64,
    
    /// K policy configuration
    pub k_policy: String,
    
    /// WASM module digest
    pub wasm_digest: String,
    
    /// Binning hash
    pub binning_hash: String,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ParityReportData {
    /// ‖ŷ_rust−ŷ_ts‖∞ parity measure
    pub rust_ts_parity_l_infinity: f64,
    
    /// |ΔECE| difference
    pub ece_delta: f64,
    
    /// Bin count verification
    pub bin_counts_identical: bool,
    
    /// Additional parity metrics
    pub additional_metrics: HashMap<String, f64>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WeeklyDriftPackData {
    /// AECE metric
    pub aece: f64,
    
    /// DECE metric
    pub dece: f64,
    
    /// Brier score
    pub brier: f64,
    
    /// Alpha value
    pub alpha: f64,
    
    /// Clamp rate percentage
    pub clamp_rate_percent: f64,
    
    /// Merged bin percentage
    pub merged_bin_percent: f64,
    
    /// Data collection week
    pub week_timestamp: SystemTime,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ReleaseBinding {
    /// Git commit hash
    pub git_commit: String,
    
    /// Build timestamp
    pub build_timestamp: SystemTime,
    
    /// Release version
    pub release_version: String,
    
    /// Environment identifier
    pub environment: String,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CryptographicAttestation {
    /// Digital signature of the fingerprint
    pub signature: String,
    
    /// Signing algorithm used
    pub algorithm: String,
    
    /// Signing timestamp
    pub timestamp: SystemTime,
    
    /// Verification public key hash
    pub public_key_hash: String,
}

impl GreenFingerprintPublisher {
    pub fn new(fingerprint_publisher: FingerprintPublisher, attestation_config: AttestationConfig) -> Self {
        Self {
            fingerprint_publisher,
            attestation_config,
        }
    }
    
    /// Generate and publish green fingerprint with cryptographic attestation
    pub async fn publish_green_fingerprint(
        &self,
        manifest: &CalibrationManifest,
        parity_report: &ParityReport,
    ) -> Result<ProductionFingerprint, ActivationError> {
        info!("🔐 Generating production green fingerprint with attestation");
        
        // Collect calibration manifest data
        let calibration_manifest = CalibrationManifestData {
            coefficients: manifest.coefficients.clone(),
            epsilon: manifest.epsilon,
            k_policy: format!("{:?}", manifest.k_policy),
            wasm_digest: manifest.wasm_digest.clone(),
            binning_hash: manifest.binning_hash.clone(),
        };
        
        // Collect parity report data
        let parity_report_data = ParityReportData {
            rust_ts_parity_l_infinity: parity_report.rust_ts_parity_l_infinity,
            ece_delta: parity_report.ece_delta,
            bin_counts_identical: parity_report.bin_counts_identical,
            additional_metrics: parity_report.additional_metrics.clone(),
        };
        
        // Generate weekly drift pack (simulated for production activation)
        let weekly_drift_pack = self.generate_weekly_drift_pack().await?;
        
        // Create release binding
        let release_binding = self.create_release_binding().await?;
        
        // Create production fingerprint
        let mut production_fingerprint = ProductionFingerprint {
            calibration_manifest,
            parity_report: parity_report_data,
            weekly_drift_pack,
            release_binding,
            attestation: CryptographicAttestation {
                signature: String::new(),
                algorithm: "RSA-SHA256".to_string(),
                timestamp: SystemTime::now(),
                public_key_hash: "placeholder_hash".to_string(),
            },
        };
        
        // Generate cryptographic attestation
        let attestation = self.generate_attestation(&production_fingerprint).await?;
        production_fingerprint.attestation = attestation;
        
        info!("✅ Production green fingerprint generated and attested");
        
        Ok(production_fingerprint)
    }
    
    async fn generate_weekly_drift_pack(&self) -> Result<WeeklyDriftPackData, ActivationError> {
        // In production, these would be actual drift metrics from the past week
        Ok(WeeklyDriftPackData {
            aece: 0.008,
            dece: 0.012,
            brier: 0.085,
            alpha: 0.15,
            clamp_rate_percent: 2.1,
            merged_bin_percent: 1.8,
            week_timestamp: SystemTime::now(),
        })
    }
    
    async fn create_release_binding(&self) -> Result<ReleaseBinding, ActivationError> {
        // In production, these would be actual release metadata
        Ok(ReleaseBinding {
            git_commit: "abc123def456".to_string(),
            build_timestamp: SystemTime::now(),
            release_version: "calib_v22.1.0".to_string(),
            environment: "production".to_string(),
        })
    }
    
    async fn generate_attestation(&self, fingerprint: &ProductionFingerprint) -> Result<CryptographicAttestation, ActivationError> {
        // In production, this would generate actual cryptographic signature
        // For now, simulate the attestation process
        
        let fingerprint_data = serde_json::to_string(fingerprint)
            .map_err(|e| ActivationError::AttestationError(format!("Serialization failed: {}", e)))?;
        
        // Simulate digital signature generation
        let signature = format!("sig_{}_{}_{}", 
            fingerprint_data.len(),
            fingerprint.release_binding.git_commit,
            fingerprint.calibration_manifest.binning_hash);
        
        Ok(CryptographicAttestation {
            signature,
            algorithm: "RSA-SHA256".to_string(),
            timestamp: SystemTime::now(),
            public_key_hash: "sha256_hash_of_public_key".to_string(),
        })
    }
}

impl Default for CanaryConfig {
    fn default() -> Self {
        Self {
            traffic_progression: vec![
                TrafficStage {
                    traffic_percentage: 5.0,
                    duration: Duration::from_secs(6 * 3600), // 6 hours
                    bucket_strategy: BucketStrategy::HashModulo { modulo: 20, remainder: 0 },
                },
                TrafficStage {
                    traffic_percentage: 25.0,
                    duration: Duration::from_secs(6 * 3600), // 6 hours
                    bucket_strategy: BucketStrategy::HashModulo { modulo: 4, remainder: 0 },
                },
                TrafficStage {
                    traffic_percentage: 50.0,
                    duration: Duration::from_secs(6 * 3600), // 6 hours
                    bucket_strategy: BucketStrategy::HashModulo { modulo: 2, remainder: 0 },
                },
                TrafficStage {
                    traffic_percentage: 100.0,
                    duration: Duration::from_secs(6 * 3600), // 6 hours
                    bucket_strategy: BucketStrategy::HashModulo { modulo: 1, remainder: 0 },
                },
            ],
            gate_thresholds: GateThresholds::default(),
            monitoring_config: MonitoringConfig::default(),
            revert_config: RevertConfig::default(),
        }
    }
}

impl Default for GateThresholds {
    fn default() -> Self {
        Self {
            p99_latency_ms: 1.0,
            aece_tau_threshold: 0.01,
            confidence_shift_threshold: 0.02,
            sla_recall_delta_threshold: 0.0,
            consecutive_breach_tolerance: 2,
            breach_window: Duration::from_secs(15 * 60), // 15 minutes
        }
    }
}

impl Default for MonitoringConfig {
    fn default() -> Self {
        Self {
            gate_evaluation_interval: Duration::from_secs(5 * 60), // 5 minutes
            smoke_probe_interval: Duration::from_secs(10 * 60), // 10 minutes
            metrics_window: Duration::from_secs(15 * 60), // 15 minutes
            alert_cooldown: Duration::from_secs(5 * 60), // 5 minutes
        }
    }
}

impl Default for RevertConfig {
    fn default() -> Self {
        Self {
            auto_revert_enabled: true,
            revert_timeout: Duration::from_secs(60), // 1 minute
            post_revert_validation: Duration::from_secs(10 * 60), // 10 minutes
        }
    }
}

impl Default for CanaryState {
    fn default() -> Self {
        let now = SystemTime::now();
        Self {
            current_stage: 0,
            stage_start_time: now,
            deployment_start_time: now,
            gate_status: GateStatus::default(),
            consecutive_breaches: 0,
            last_gate_evaluation: now,
            deployment_status: CanaryDeploymentStatus::NotStarted,
            error_history: Vec::new(),
        }
    }
}

impl Default for GateStatus {
    fn default() -> Self {
        let now = SystemTime::now();
        Self {
            p99_latency: GateResult {
                passed: false,
                measured_value: 0.0,
                threshold_value: 0.0,
                timestamp: now,
                context: None,
            },
            aece_tau_gates: HashMap::new(),
            confidence_shift: GateResult {
                passed: false,
                measured_value: 0.0,
                threshold_value: 0.0,
                timestamp: now,
                context: None,
            },
            sla_recall_delta: GateResult {
                passed: false,
                measured_value: 0.0,
                threshold_value: 0.0,
                timestamp: now,
                context: None,
            },
            overall_passed: false,
            evaluation_timestamp: now,
        }
    }
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CanaryError {
    pub timestamp: SystemTime,
    pub error_type: String,
    pub message: String,
    pub stage: u32,
}

#[derive(Debug, Error)]
pub enum ActivationError {
    #[error("Configuration error: {0}")]
    ConfigError(String),
    
    #[error("Gate evaluation failed: {0}")]
    GateEvaluationError(String),
    
    #[error("Metrics collection failed: {0}")]
    MetricsError(String),
    
    #[error("Smoke probe failed: {0}")]
    SmokeProbeError(String),
    
    #[error("Fingerprint generation failed: {0}")]
    FingerprintError(String),
    
    #[error("Attestation error: {0}")]
    AttestationError(String),
    
    #[error("Deployment error: {0}")]
    DeploymentError(String),
    
    #[error("Revert operation failed: {0}")]
    RevertError(String),
}

impl ProductionActivationController {
    pub fn new(
        config: CanaryConfig,
        sla_monitor: Arc<SlaMonitor>,
        fingerprint_publisher: FingerprintPublisher,
    ) -> Self {
        let gate_enforcer = CanaryGateEnforcer::new(
            config.gate_thresholds.clone(),
            config.monitoring_config.clone(),
            Arc::clone(&sla_monitor),
        );
        
        let smoke_prober = SmokeProbeExecutor::new(SmokeProbeConfig {
            execution_interval: config.monitoring_config.smoke_probe_interval,
            probe_timeout: Duration::from_secs(30),
            probes_per_execution: 4,
            failure_threshold: 0.8,
        });
        
        Self {
            config,
            state: Arc::new(tokio::sync::RwLock::new(CanaryState::default())),
            gate_enforcer,
            fingerprint_publisher,
            sla_monitor,
            smoke_prober,
        }
    }
    
    /// Start the 24-hour canary deployment with 4-gate progression
    pub async fn start_24h_canary(&self) -> Result<(), ActivationError> {
        info!("🚀 Starting CALIB_V22 24-hour canary deployment");
        
        // Initialize deployment state
        {
            let mut state = self.state.write().await;
            state.deployment_start_time = SystemTime::now();
            state.deployment_status = CanaryDeploymentStatus::Running { stage: 0, progress_percent: 0.0 };
            state.current_stage = 0;
            state.stage_start_time = SystemTime::now();
        }
        
        // Execute canary stages with gate enforcement
        for (stage_index, stage) in self.config.traffic_progression.iter().enumerate() {
            info!("📈 Starting canary stage {} - {}% traffic", stage_index + 1, stage.traffic_percentage);
            
            // Update state
            {
                let mut state = self.state.write().await;
                state.current_stage = stage_index as u32 + 1;
                state.stage_start_time = SystemTime::now();
                state.deployment_status = CanaryDeploymentStatus::Running { 
                    stage: stage_index as u32 + 1, 
                    progress_percent: 0.0 
                };
            }
            
            // Run stage with continuous monitoring
            self.execute_canary_stage(stage_index as u32 + 1, stage).await?;
            
            info!("✅ Canary stage {} completed successfully", stage_index + 1);
        }
        
        // Mark deployment as completed
        {
            let mut state = self.state.write().await;
            state.deployment_status = CanaryDeploymentStatus::Completed;
        }
        
        info!("🎉 24-hour canary deployment completed successfully");
        Ok(())
    }
    
    async fn execute_canary_stage(&self, stage_number: u32, stage: &TrafficStage) -> Result<(), ActivationError> {
        let stage_start = SystemTime::now();
        let stage_end = stage_start + stage.duration;
        
        let mut consecutive_failures = 0;
        let mut last_gate_evaluation = SystemTime::now();
        
        // Stage monitoring loop
        while SystemTime::now() < stage_end {
            // Check if it's time for gate evaluation
            if SystemTime::now().duration_since(last_gate_evaluation).unwrap() 
                >= self.config.monitoring_config.gate_evaluation_interval {
                
                // Evaluate canary gates
                match self.gate_enforcer.evaluate_gates(stage_number).await {
                    Ok(gate_status) => {
                        // Update state with gate results
                        {
                            let mut state = self.state.write().await;
                            state.gate_status = gate_status.clone();
                            state.last_gate_evaluation = SystemTime::now();
                            
                            if !gate_status.overall_passed {
                                state.consecutive_breaches += 1;
                                consecutive_failures += 1;
                            } else {
                                state.consecutive_breaches = 0;
                                consecutive_failures = 0;
                            }
                        }
                        
                        // Check for auto-revert condition
                        if consecutive_failures >= self.config.gate_thresholds.consecutive_breach_tolerance {
                            warn!("🚨 Gate failures exceeded tolerance - initiating auto-revert");
                            self.execute_auto_revert(stage_number, "Gate failure threshold exceeded").await?;
                            return Err(ActivationError::DeploymentError(
                                "Canary deployment failed - auto-reverted due to gate failures".to_string()
                            ));
                        }
                        
                        last_gate_evaluation = SystemTime::now();
                    }
                    Err(e) => {
                        error!("❌ Gate evaluation failed: {}", e);
                        // Continue monitoring but log the error
                        let mut state = self.state.write().await;
                        state.error_history.push(CanaryError {
                            timestamp: SystemTime::now(),
                            error_type: "GateEvaluationError".to_string(),
                            message: e.to_string(),
                            stage: stage_number,
                        });
                    }
                }
            }
            
            // Execute smoke probes periodically
            if SystemTime::now().duration_since(stage_start).unwrap().as_secs() % 
                self.config.monitoring_config.smoke_probe_interval.as_secs() == 0 {
                
                match self.smoke_prober.execute_smoke_probes().await {
                    Ok(probe_result) => {
                        if probe_result.overall_success_rate < 0.8 {
                            warn!("⚠️  Smoke probe success rate below threshold: {:.1}%", 
                                  probe_result.overall_success_rate * 100.0);
                        }
                    }
                    Err(e) => {
                        warn!("❌ Smoke probe execution failed: {}", e);
                    }
                }
            }
            
            // Update progress
            let elapsed = SystemTime::now().duration_since(stage_start).unwrap();
            let progress_percent = (elapsed.as_secs_f64() / stage.duration.as_secs_f64() * 100.0).min(100.0);
            
            {
                let mut state = self.state.write().await;
                state.deployment_status = CanaryDeploymentStatus::Running { 
                    stage: stage_number, 
                    progress_percent 
                };
            }
            
            // Sleep before next monitoring cycle
            sleep(Duration::from_secs(30)).await;
        }
        
        Ok(())
    }
    
    async fn execute_auto_revert(&self, failed_stage: u32, reason: &str) -> Result<(), ActivationError> {
        warn!("🔄 Executing auto-revert for stage {} - Reason: {}", failed_stage, reason);
        
        // Update state to reverting
        {
            let mut state = self.state.write().await;
            state.deployment_status = CanaryDeploymentStatus::Reverting;
        }
        
        // Execute revert logic (this would disable CALIB_V22 flag and rollback)
        // In production, this would:
        // 1. Set CALIB_V22=false for all affected repos
        // 2. Clear calibrated caches
        // 3. Restore previous stable configuration
        
        sleep(self.config.revert_config.revert_timeout).await;
        
        // Validate revert success
        sleep(self.config.revert_config.post_revert_validation).await;
        
        // Update state to failed with revert completed
        {
            let mut state = self.state.write().await;
            state.deployment_status = CanaryDeploymentStatus::Failed { 
                reason: reason.to_string(), 
                reverted: true 
            };
        }
        
        warn!("✅ Auto-revert completed successfully");
        Ok(())
    }
    
    /// Get current deployment status
    pub async fn get_deployment_status(&self) -> CanaryDeploymentStatus {
        let state = self.state.read().await;
        state.deployment_status.clone()
    }
    
    /// Get comprehensive deployment report
    pub async fn get_deployment_report(&self) -> Result<CanaryDeploymentReport, ActivationError> {
        let state = self.state.read().await;
        
        Ok(CanaryDeploymentReport {
            deployment_status: state.deployment_status.clone(),
            current_stage: state.current_stage,
            deployment_duration: SystemTime::now().duration_since(state.deployment_start_time).unwrap(),
            stage_duration: SystemTime::now().duration_since(state.stage_start_time).unwrap(),
            gate_status: state.gate_status.clone(),
            consecutive_breaches: state.consecutive_breaches,
            error_history: state.error_history.clone(),
            total_stages: self.config.traffic_progression.len() as u32,
        })
    }
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CanaryDeploymentReport {
    pub deployment_status: CanaryDeploymentStatus,
    pub current_stage: u32,
    pub deployment_duration: Duration,
    pub stage_duration: Duration,
    pub gate_status: GateStatus,
    pub consecutive_breaches: u32,
    pub error_history: Vec<CanaryError>,
    pub total_stages: u32,
}

#[cfg(test)]
mod tests {
    use super::*;
    
    #[test]
    fn test_canary_config_default() {
        let config = CanaryConfig::default();
        assert_eq!(config.traffic_progression.len(), 4);
        assert_eq!(config.traffic_progression[0].traffic_percentage, 5.0);
        assert_eq!(config.traffic_progression[3].traffic_percentage, 100.0);
    }
    
    #[test]
    fn test_gate_thresholds() {
        let thresholds = GateThresholds::default();
        assert_eq!(thresholds.p99_latency_ms, 1.0);
        assert_eq!(thresholds.aece_tau_threshold, 0.01);
    }
    
    #[tokio::test]
    async fn test_smoke_probe_executor() {
        let config = SmokeProbeConfig {
            execution_interval: Duration::from_secs(60),
            probe_timeout: Duration::from_secs(10),
            probes_per_execution: 4,
            failure_threshold: 0.8,
        };
        
        let executor = SmokeProbeExecutor::new(config);
        let result = executor.execute_smoke_probes().await.unwrap();
        
        assert!(result.overall_success_rate > 0.0);
        assert!(result.identity_calibrated.latency_ms > 0.0);
    }
}