rexis_rag/incremental/

integrity.rs

//! # Integrity Checker
//!
//! Comprehensive integrity checking and validation for incremental indexing systems.
//! Ensures consistency, detects corruption, and provides health monitoring.

use crate::RragResult;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::sync::Arc;
use tokio::sync::RwLock;
use uuid::Uuid;

/// Integrity checker configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct IntegrityConfig {
    /// Enable automatic integrity checks
    pub enable_auto_checks: bool,

    /// Check interval in seconds
    pub check_interval_secs: u64,

    /// Comprehensive check interval in seconds
    pub comprehensive_check_interval_secs: u64,

    /// Enable consistency validation
    pub enable_consistency_checks: bool,

    /// Enable corruption detection
    pub enable_corruption_detection: bool,

    /// Enable performance monitoring
    pub enable_performance_monitoring: bool,

    /// Maximum repair attempts
    pub max_repair_attempts: u32,

    /// Enable automatic repairs
    pub enable_auto_repair: bool,

    /// Health check thresholds
    pub health_thresholds: HealthThresholds,
}

/// Health check thresholds
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct HealthThresholds {
    /// Maximum allowed error rate (0.0 to 1.0)
    pub max_error_rate: f64,

    /// Maximum response time in milliseconds
    pub max_response_time_ms: u64,

    /// Minimum success rate (0.0 to 1.0)
    pub min_success_rate: f64,

    /// Maximum queue depth
    pub max_queue_depth: usize,

    /// Maximum memory usage in MB
    pub max_memory_usage_mb: f64,

    /// Maximum storage usage percentage
    pub max_storage_usage_percent: f64,
}

impl Default for IntegrityConfig {
    fn default() -> Self {
        Self {
            enable_auto_checks: true,
            check_interval_secs: 300,                // 5 minutes
            comprehensive_check_interval_secs: 3600, // 1 hour
            enable_consistency_checks: true,
            enable_corruption_detection: true,
            enable_performance_monitoring: true,
            max_repair_attempts: 3,
            enable_auto_repair: true,
            health_thresholds: HealthThresholds::default(),
        }
    }
}

impl Default for HealthThresholds {
    fn default() -> Self {
        Self {
            max_error_rate: 0.05,        // 5%
            max_response_time_ms: 10000, // 10 seconds
            min_success_rate: 0.95,      // 95%
            max_queue_depth: 1000,
            max_memory_usage_mb: 1024.0,     // 1GB
            max_storage_usage_percent: 80.0, // 80%
        }
    }
}

/// Types of integrity errors
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub enum IntegrityError {
    /// Hash mismatch detected
    HashMismatch {
        expected: String,
        actual: String,
        entity_id: String,
    },

    /// Missing reference
    MissingReference {
        reference_id: String,
        referenced_by: String,
    },

    /// Orphaned data
    OrphanedData {
        entity_id: String,
        entity_type: String,
    },

    /// Version inconsistency
    VersionInconsistency {
        entity_id: String,
        expected_version: u64,
        actual_version: u64,
    },

    /// Index corruption
    IndexCorruption {
        index_name: String,
        corruption_type: String,
        details: String,
    },

    /// Data size mismatch
    SizeMismatch {
        entity_id: String,
        expected_size: u64,
        actual_size: u64,
    },

    /// Timestamp inconsistency
    TimestampInconsistency { entity_id: String, issue: String },

    /// Duplicate entries
    DuplicateEntries {
        entity_ids: Vec<String>,
        duplicate_field: String,
    },
}

/// Consistency report
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ConsistencyReport {
    /// Report ID
    pub report_id: String,

    /// Report generation timestamp
    pub generated_at: chrono::DateTime<chrono::Utc>,

    /// Report type
    pub report_type: ReportType,

    /// Overall health status
    pub overall_health: HealthStatus,

    /// Detected integrity errors
    pub integrity_errors: Vec<IntegrityError>,

    /// Performance metrics
    pub performance_metrics: PerformanceMetrics,

    /// System statistics
    pub system_stats: SystemStats,

    /// Recommendations
    pub recommendations: Vec<Recommendation>,

    /// Check duration in milliseconds
    pub check_duration_ms: u64,

    /// Entities checked
    pub entities_checked: usize,

    /// Repair actions taken
    pub repair_actions: Vec<RepairAction>,
}

/// Report types
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum ReportType {
    /// Quick health check
    Quick,
    /// Comprehensive integrity check
    Comprehensive,
    /// Targeted check for specific issues
    Targeted(String),
    /// Emergency repair validation
    Emergency,
}

/// Health status levels
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub enum HealthStatus {
    /// All systems healthy
    Healthy,
    /// Minor issues detected
    Warning,
    /// Significant issues requiring attention
    Critical,
    /// System compromised, immediate action required
    Emergency,
}

/// Performance metrics
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PerformanceMetrics {
    /// Average response time in milliseconds
    pub avg_response_time_ms: f64,

    /// 95th percentile response time
    pub p95_response_time_ms: f64,

    /// 99th percentile response time
    pub p99_response_time_ms: f64,

    /// Operations per second
    pub operations_per_second: f64,

    /// Error rate (0.0 to 1.0)
    pub error_rate: f64,

    /// Success rate (0.0 to 1.0)
    pub success_rate: f64,

    /// Memory usage in MB
    pub memory_usage_mb: f64,

    /// CPU usage percentage
    pub cpu_usage_percent: f64,

    /// Storage usage in bytes
    pub storage_usage_bytes: u64,
}

/// System statistics
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SystemStats {
    /// Total documents indexed
    pub total_documents: usize,

    /// Total chunks processed
    pub total_chunks: usize,

    /// Total embeddings stored
    pub total_embeddings: usize,

    /// Index counts by type
    pub index_counts: HashMap<String, usize>,

    /// Storage distribution
    pub storage_distribution: HashMap<String, u64>,

    /// System uptime in seconds
    pub uptime_seconds: u64,

    /// Last maintenance timestamp
    pub last_maintenance_at: Option<chrono::DateTime<chrono::Utc>>,
}

/// System recommendations
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Recommendation {
    /// Recommendation ID
    pub recommendation_id: String,

    /// Recommendation type
    pub recommendation_type: RecommendationType,

    /// Priority level
    pub priority: RecommendationPriority,

    /// Description
    pub description: String,

    /// Suggested actions
    pub suggested_actions: Vec<String>,

    /// Expected impact
    pub expected_impact: String,

    /// Estimated effort
    pub estimated_effort: String,
}

/// Types of recommendations
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum RecommendationType {
    /// Performance optimization
    Performance,
    /// Storage optimization
    Storage,
    /// Security improvement
    Security,
    /// Maintenance task
    Maintenance,
    /// Capacity planning
    Capacity,
    /// Configuration adjustment
    Configuration,
}

/// Recommendation priorities
#[derive(Debug, Clone, Serialize, Deserialize, PartialOrd, PartialEq)]
pub enum RecommendationPriority {
    Low = 1,
    Medium = 2,
    High = 3,
    Critical = 4,
}

/// Repair actions
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RepairAction {
    /// Action ID
    pub action_id: String,

    /// Action type
    pub action_type: RepairActionType,

    /// Target entity
    pub target_entity: String,

    /// Action description
    pub description: String,

    /// Action timestamp
    pub executed_at: chrono::DateTime<chrono::Utc>,

    /// Action result
    pub result: RepairResult,

    /// Details about the repair
    pub details: HashMap<String, serde_json::Value>,
}

/// Types of repair actions
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum RepairActionType {
    /// Rebuild index
    RebuildIndex,
    /// Fix hash mismatch
    FixHashMismatch,
    /// Remove orphaned data
    RemoveOrphanedData,
    /// Update version
    UpdateVersion,
    /// Repair corruption
    RepairCorruption,
    /// Clean duplicates
    CleanDuplicates,
    /// Restore from backup
    RestoreFromBackup,
}

/// Repair operation results
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum RepairResult {
    /// Repair successful
    Success,
    /// Repair failed
    Failed(String),
    /// Repair partially successful
    Partial(String),
    /// Repair skipped
    Skipped(String),
}

/// Validation result for specific checks
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ValidationResult {
    /// Validation check name
    pub check_name: String,

    /// Whether validation passed
    pub passed: bool,

    /// Validation details
    pub details: String,

    /// Entities validated
    pub entities_validated: usize,

    /// Validation duration
    pub validation_duration_ms: u64,

    /// Issues found
    pub issues_found: Vec<IntegrityError>,
}

/// Health metrics for monitoring
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct HealthMetrics {
    /// Current health status
    pub current_health: HealthStatus,

    /// Health history over time
    pub health_history: Vec<HealthDataPoint>,

    /// Alert conditions met
    pub active_alerts: Vec<AlertCondition>,

    /// System vitals
    pub vitals: SystemVitals,

    /// Last health check timestamp
    pub last_check_at: chrono::DateTime<chrono::Utc>,
}

/// Health data point for trending
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct HealthDataPoint {
    /// Timestamp
    pub timestamp: chrono::DateTime<chrono::Utc>,

    /// Health status at this time
    pub health_status: HealthStatus,

    /// Key metrics at this time
    pub metrics: HashMap<String, f64>,
}

/// Alert conditions
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AlertCondition {
    /// Alert ID
    pub alert_id: String,

    /// Alert type
    pub alert_type: String,

    /// Alert severity
    pub severity: HealthStatus,

    /// Alert description
    pub description: String,

    /// When alert was triggered
    pub triggered_at: chrono::DateTime<chrono::Utc>,

    /// Alert metadata
    pub metadata: HashMap<String, serde_json::Value>,
}

/// System vitals
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SystemVitals {
    /// Memory usage
    pub memory_usage_percent: f64,

    /// Storage usage
    pub storage_usage_percent: f64,

    /// CPU usage
    pub cpu_usage_percent: f64,

    /// Network usage
    pub network_usage_bytes_per_second: f64,

    /// Queue depths
    pub queue_depths: HashMap<String, usize>,

    /// Connection counts
    pub active_connections: usize,
}

/// Main integrity checker
pub struct IntegrityChecker {
    /// Configuration
    config: IntegrityConfig,

    /// Check history
    check_history: Arc<RwLock<Vec<ConsistencyReport>>>,

    /// Health metrics
    health_metrics: Arc<RwLock<HealthMetrics>>,

    /// Active repair operations
    active_repairs: Arc<RwLock<HashMap<String, RepairAction>>>,

    /// Statistics
    stats: Arc<RwLock<IntegrityStats>>,

    /// Background task handles
    task_handles: Arc<tokio::sync::Mutex<Vec<tokio::task::JoinHandle<()>>>>,
}

/// Integrity checker statistics
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct IntegrityStats {
    /// Total checks performed
    pub total_checks: u64,

    /// Quick checks performed
    pub quick_checks: u64,

    /// Comprehensive checks performed
    pub comprehensive_checks: u64,

    /// Total integrity errors found
    pub total_errors_found: u64,

    /// Errors by type
    pub errors_by_type: HashMap<String, u64>,

    /// Total repairs attempted
    pub total_repairs_attempted: u64,

    /// Successful repairs
    pub successful_repairs: u64,

    /// Failed repairs
    pub failed_repairs: u64,

    /// Average check duration
    pub avg_check_duration_ms: f64,

    /// System availability
    pub system_availability_percent: f64,

    /// Last updated
    pub last_updated: chrono::DateTime<chrono::Utc>,
}

impl IntegrityChecker {
    /// Create new integrity checker
    pub async fn new(config: IntegrityConfig) -> RragResult<Self> {
        let checker = Self {
            config: config.clone(),
            check_history: Arc::new(RwLock::new(Vec::new())),
            health_metrics: Arc::new(RwLock::new(HealthMetrics {
                current_health: HealthStatus::Healthy,
                health_history: Vec::new(),
                active_alerts: Vec::new(),
                vitals: SystemVitals {
                    memory_usage_percent: 0.0,
                    storage_usage_percent: 0.0,
                    cpu_usage_percent: 0.0,
                    network_usage_bytes_per_second: 0.0,
                    queue_depths: HashMap::new(),
                    active_connections: 0,
                },
                last_check_at: chrono::Utc::now(),
            })),
            active_repairs: Arc::new(RwLock::new(HashMap::new())),
            stats: Arc::new(RwLock::new(IntegrityStats {
                total_checks: 0,
                quick_checks: 0,
                comprehensive_checks: 0,
                total_errors_found: 0,
                errors_by_type: HashMap::new(),
                total_repairs_attempted: 0,
                successful_repairs: 0,
                failed_repairs: 0,
                avg_check_duration_ms: 0.0,
                system_availability_percent: 100.0,
                last_updated: chrono::Utc::now(),
            })),
            task_handles: Arc::new(tokio::sync::Mutex::new(Vec::new())),
        };

        if config.enable_auto_checks {
            checker.start_background_tasks().await?;
        }

        Ok(checker)
    }

    /// Perform quick integrity check
    pub async fn quick_check(&self) -> RragResult<ConsistencyReport> {
        let start_time = std::time::Instant::now();
        let report_id = Uuid::new_v4().to_string();

        // Perform basic integrity checks
        let mut integrity_errors = Vec::new();
        let mut repair_actions = Vec::new();

        // Check for basic inconsistencies
        let basic_errors = self.check_basic_consistency().await?;
        integrity_errors.extend(basic_errors);

        // Check performance metrics
        let performance_metrics = self.collect_performance_metrics().await?;
        let system_stats = self.collect_system_stats().await?;

        // Determine overall health
        let overall_health = self
            .determine_health_status(&integrity_errors, &performance_metrics)
            .await?;

        // Generate recommendations
        let recommendations = self
            .generate_recommendations(&integrity_errors, &performance_metrics, &overall_health)
            .await?;

        // Perform automatic repairs if enabled
        if self.config.enable_auto_repair && !integrity_errors.is_empty() {
            repair_actions = self.perform_auto_repairs(&integrity_errors).await?;
        }

        let check_duration = start_time.elapsed().as_millis() as u64;

        let report = ConsistencyReport {
            report_id,
            generated_at: chrono::Utc::now(),
            report_type: ReportType::Quick,
            overall_health,
            integrity_errors,
            performance_metrics,
            system_stats,
            recommendations,
            check_duration_ms: check_duration,
            entities_checked: 100, // Would be actual count
            repair_actions,
        };

        // Update statistics and history
        self.update_check_statistics(&report).await?;
        self.add_to_history(report.clone()).await?;

        Ok(report)
    }

    /// Perform comprehensive integrity check
    pub async fn comprehensive_check(&self) -> RragResult<ConsistencyReport> {
        let start_time = std::time::Instant::now();
        let report_id = Uuid::new_v4().to_string();

        let mut integrity_errors = Vec::new();
        let mut repair_actions = Vec::new();

        // Comprehensive checks
        let basic_errors = self.check_basic_consistency().await?;
        let hash_errors = self.check_hash_integrity().await?;
        let reference_errors = self.check_reference_integrity().await?;
        let version_errors = self.check_version_consistency().await?;
        let index_errors = self.check_index_integrity().await?;

        integrity_errors.extend(basic_errors);
        integrity_errors.extend(hash_errors);
        integrity_errors.extend(reference_errors);
        integrity_errors.extend(version_errors);
        integrity_errors.extend(index_errors);

        let performance_metrics = self.collect_performance_metrics().await?;
        let system_stats = self.collect_system_stats().await?;
        let overall_health = self
            .determine_health_status(&integrity_errors, &performance_metrics)
            .await?;
        let recommendations = self
            .generate_recommendations(&integrity_errors, &performance_metrics, &overall_health)
            .await?;

        if self.config.enable_auto_repair && !integrity_errors.is_empty() {
            repair_actions = self.perform_auto_repairs(&integrity_errors).await?;
        }

        let check_duration = start_time.elapsed().as_millis() as u64;

        let report = ConsistencyReport {
            report_id,
            generated_at: chrono::Utc::now(),
            report_type: ReportType::Comprehensive,
            overall_health,
            integrity_errors,
            performance_metrics,
            system_stats,
            recommendations,
            check_duration_ms: check_duration,
            entities_checked: 1000, // Would be actual count
            repair_actions,
        };

        self.update_check_statistics(&report).await?;
        self.add_to_history(report.clone()).await?;

        Ok(report)
    }

    /// Get current health metrics
    pub async fn get_health_metrics(&self) -> HealthMetrics {
        self.health_metrics.read().await.clone()
    }

    /// Get integrity statistics
    pub async fn get_stats(&self) -> IntegrityStats {
        self.stats.read().await.clone()
    }

    /// Get check history
    pub async fn get_check_history(
        &self,
        limit: Option<usize>,
    ) -> RragResult<Vec<ConsistencyReport>> {
        let history = self.check_history.read().await;
        let limit = limit.unwrap_or(history.len());
        Ok(history.iter().rev().take(limit).cloned().collect())
    }

    /// Health check
    pub async fn health_check(&self) -> RragResult<bool> {
        let handles = self.task_handles.lock().await;
        let all_running = handles.iter().all(|handle| !handle.is_finished());

        let metrics = self.get_health_metrics().await;
        let healthy_status = matches!(
            metrics.current_health,
            HealthStatus::Healthy | HealthStatus::Warning
        );

        Ok(all_running && healthy_status)
    }

    /// Start background monitoring tasks
    async fn start_background_tasks(&self) -> RragResult<()> {
        let mut handles = self.task_handles.lock().await;

        // Quick check task
        handles.push(self.start_quick_check_task().await);

        // Comprehensive check task
        handles.push(self.start_comprehensive_check_task().await);

        // Health monitoring task
        if self.config.enable_performance_monitoring {
            handles.push(self.start_health_monitoring_task().await);
        }

        Ok(())
    }

    /// Start quick check background task
    async fn start_quick_check_task(&self) -> tokio::task::JoinHandle<()> {
        let checker = self.clone_for_task();
        let interval = self.config.check_interval_secs;

        tokio::spawn(async move {
            let mut timer = tokio::time::interval(tokio::time::Duration::from_secs(interval));

            loop {
                timer.tick().await;

                if let Err(e) = checker.quick_check().await {
                    tracing::debug!("Quick integrity check failed: {}", e);
                }
            }
        })
    }

    /// Start comprehensive check background task
    async fn start_comprehensive_check_task(&self) -> tokio::task::JoinHandle<()> {
        let checker = self.clone_for_task();
        let interval = self.config.comprehensive_check_interval_secs;

        tokio::spawn(async move {
            let mut timer = tokio::time::interval(tokio::time::Duration::from_secs(interval));

            loop {
                timer.tick().await;

                if let Err(e) = checker.comprehensive_check().await {
                    tracing::debug!("Comprehensive integrity check failed: {}", e);
                }
            }
        })
    }

    /// Start health monitoring task
    async fn start_health_monitoring_task(&self) -> tokio::task::JoinHandle<()> {
        let health_metrics = Arc::clone(&self.health_metrics);

        tokio::spawn(async move {
            let mut timer = tokio::time::interval(tokio::time::Duration::from_secs(30));

            loop {
                timer.tick().await;

                // Update health metrics
                let mut metrics = health_metrics.write().await;

                // Collect current vitals (placeholder implementation)
                metrics.vitals = SystemVitals {
                    memory_usage_percent: 45.0, // Would be actual measurement
                    storage_usage_percent: 60.0,
                    cpu_usage_percent: 25.0,
                    network_usage_bytes_per_second: 1024.0,
                    queue_depths: HashMap::new(),
                    active_connections: 10,
                };

                metrics.last_check_at = chrono::Utc::now();

                // Add data point to history
                let data_point = HealthDataPoint {
                    timestamp: chrono::Utc::now(),
                    health_status: metrics.current_health.clone(),
                    metrics: HashMap::new(), // Would include actual metrics
                };

                metrics.health_history.push(data_point);

                // Limit history size
                if metrics.health_history.len() > 1000 {
                    metrics.health_history.remove(0);
                }
            }
        })
    }

    /// Clone checker for background tasks (simplified)
    fn clone_for_task(&self) -> Self {
        // In a real implementation, this would properly clone or use Arc references
        Self {
            config: self.config.clone(),
            check_history: Arc::clone(&self.check_history),
            health_metrics: Arc::clone(&self.health_metrics),
            active_repairs: Arc::clone(&self.active_repairs),
            stats: Arc::clone(&self.stats),
            task_handles: Arc::clone(&self.task_handles),
        }
    }

    // Check implementations (simplified placeholders)
    async fn check_basic_consistency(&self) -> RragResult<Vec<IntegrityError>> {
        // Would perform actual basic consistency checks
        Ok(Vec::new())
    }

    async fn check_hash_integrity(&self) -> RragResult<Vec<IntegrityError>> {
        // Would verify hash integrity across documents and chunks
        Ok(Vec::new())
    }

    async fn check_reference_integrity(&self) -> RragResult<Vec<IntegrityError>> {
        // Would check referential integrity between entities
        Ok(Vec::new())
    }

    async fn check_version_consistency(&self) -> RragResult<Vec<IntegrityError>> {
        // Would verify version consistency
        Ok(Vec::new())
    }

    async fn check_index_integrity(&self) -> RragResult<Vec<IntegrityError>> {
        // Would check index structures for corruption
        Ok(Vec::new())
    }

    async fn collect_performance_metrics(&self) -> RragResult<PerformanceMetrics> {
        Ok(PerformanceMetrics {
            avg_response_time_ms: 150.0,
            p95_response_time_ms: 500.0,
            p99_response_time_ms: 1000.0,
            operations_per_second: 100.0,
            error_rate: 0.01,
            success_rate: 0.99,
            memory_usage_mb: 512.0,
            cpu_usage_percent: 45.0,
            storage_usage_bytes: 1024 * 1024 * 500, // 500MB
        })
    }

    async fn collect_system_stats(&self) -> RragResult<SystemStats> {
        Ok(SystemStats {
            total_documents: 1000,
            total_chunks: 5000,
            total_embeddings: 5000,
            index_counts: HashMap::new(),
            storage_distribution: HashMap::new(),
            uptime_seconds: 86400, // 1 day
            last_maintenance_at: Some(chrono::Utc::now() - chrono::Duration::hours(12)),
        })
    }

    async fn determine_health_status(
        &self,
        errors: &[IntegrityError],
        metrics: &PerformanceMetrics,
    ) -> RragResult<HealthStatus> {
        if !errors.is_empty() {
            return Ok(HealthStatus::Critical);
        }

        if metrics.error_rate > self.config.health_thresholds.max_error_rate {
            return Ok(HealthStatus::Warning);
        }

        if metrics.success_rate < self.config.health_thresholds.min_success_rate {
            return Ok(HealthStatus::Warning);
        }

        Ok(HealthStatus::Healthy)
    }

    async fn generate_recommendations(
        &self,
        errors: &[IntegrityError],
        metrics: &PerformanceMetrics,
        _health: &HealthStatus,
    ) -> RragResult<Vec<Recommendation>> {
        let mut recommendations = Vec::new();

        if !errors.is_empty() {
            recommendations.push(Recommendation {
                recommendation_id: Uuid::new_v4().to_string(),
                recommendation_type: RecommendationType::Maintenance,
                priority: RecommendationPriority::High,
                description: "Integrity errors detected - immediate attention required".to_string(),
                suggested_actions: vec!["Run comprehensive integrity check".to_string()],
                expected_impact: "Improved system reliability".to_string(),
                estimated_effort: "Medium".to_string(),
            });
        }

        if metrics.avg_response_time_ms > 1000.0 {
            recommendations.push(Recommendation {
                recommendation_id: Uuid::new_v4().to_string(),
                recommendation_type: RecommendationType::Performance,
                priority: RecommendationPriority::Medium,
                description: "Response times are elevated".to_string(),
                suggested_actions: vec![
                    "Optimize queries".to_string(),
                    "Scale resources".to_string(),
                ],
                expected_impact: "Faster response times".to_string(),
                estimated_effort: "Low".to_string(),
            });
        }

        Ok(recommendations)
    }

    async fn perform_auto_repairs(
        &self,
        errors: &[IntegrityError],
    ) -> RragResult<Vec<RepairAction>> {
        let mut repairs = Vec::new();

        for error in errors {
            if let Some(repair) = self.attempt_repair(error).await? {
                repairs.push(repair);
            }
        }

        Ok(repairs)
    }

    async fn attempt_repair(&self, error: &IntegrityError) -> RragResult<Option<RepairAction>> {
        match error {
            IntegrityError::OrphanedData { entity_id, .. } => Some(RepairAction {
                action_id: Uuid::new_v4().to_string(),
                action_type: RepairActionType::RemoveOrphanedData,
                target_entity: entity_id.clone(),
                description: "Removed orphaned data".to_string(),
                executed_at: chrono::Utc::now(),
                result: RepairResult::Success,
                details: HashMap::new(),
            }),
            _ => None, // Other repairs would be implemented
        }
        .pipe(Ok)
    }

    async fn update_check_statistics(&self, report: &ConsistencyReport) -> RragResult<()> {
        let mut stats = self.stats.write().await;

        stats.total_checks += 1;
        match report.report_type {
            ReportType::Quick => stats.quick_checks += 1,
            ReportType::Comprehensive => stats.comprehensive_checks += 1,
            _ => {}
        }

        stats.total_errors_found += report.integrity_errors.len() as u64;

        for error in &report.integrity_errors {
            let error_type = format!("{:?}", error)
                .split('{')
                .next()
                .unwrap_or("Unknown")
                .to_string();
            *stats.errors_by_type.entry(error_type).or_insert(0) += 1;
        }

        stats.avg_check_duration_ms =
            (stats.avg_check_duration_ms + report.check_duration_ms as f64) / 2.0;

        stats.last_updated = chrono::Utc::now();

        Ok(())
    }

    async fn add_to_history(&self, report: ConsistencyReport) -> RragResult<()> {
        let mut history = self.check_history.write().await;
        history.push(report);

        // Limit history size
        if history.len() > 100 {
            history.remove(0);
        }

        Ok(())
    }
}

// Helper trait for pipe operations
trait Pipe<T> {
    fn pipe<U, F>(self, f: F) -> U
    where
        F: FnOnce(T) -> U;
}

impl<T> Pipe<T> for T {
    fn pipe<U, F>(self, f: F) -> U
    where
        F: FnOnce(T) -> U,
    {
        f(self)
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[tokio::test]
    async fn test_integrity_checker_creation() {
        let config = IntegrityConfig::default();
        let checker = IntegrityChecker::new(config).await.unwrap();
        assert!(checker.health_check().await.unwrap());
    }

    #[tokio::test]
    async fn test_quick_check() {
        let mut config = IntegrityConfig::default();
        config.enable_auto_checks = false; // Disable auto checks for test

        let checker = IntegrityChecker::new(config).await.unwrap();
        let report = checker.quick_check().await.unwrap();

        assert_eq!(report.report_type, ReportType::Quick);
        assert!(report.check_duration_ms > 0);
        assert_eq!(report.entities_checked, 100); // Test placeholder value
    }

    #[tokio::test]
    async fn test_comprehensive_check() {
        let mut config = IntegrityConfig::default();
        config.enable_auto_checks = false;

        let checker = IntegrityChecker::new(config).await.unwrap();
        let report = checker.comprehensive_check().await.unwrap();

        assert_eq!(report.report_type, ReportType::Comprehensive);
        assert!(report.check_duration_ms > 0);
        assert_eq!(report.entities_checked, 1000); // Test placeholder value
    }

    #[tokio::test]
    async fn test_health_metrics() {
        let config = IntegrityConfig::default();
        let checker = IntegrityChecker::new(config).await.unwrap();

        let metrics = checker.get_health_metrics().await;
        assert_eq!(metrics.current_health, HealthStatus::Healthy);
        assert!(metrics.last_check_at <= chrono::Utc::now());
    }

    #[tokio::test]
    async fn test_statistics() {
        let mut config = IntegrityConfig::default();
        config.enable_auto_checks = false;

        let checker = IntegrityChecker::new(config).await.unwrap();

        // Perform a check to update statistics
        checker.quick_check().await.unwrap();

        let stats = checker.get_stats().await;
        assert_eq!(stats.total_checks, 1);
        assert_eq!(stats.quick_checks, 1);
        assert_eq!(stats.comprehensive_checks, 0);
    }

    #[test]
    fn test_health_status_ordering() {
        assert!(HealthStatus::Healthy < HealthStatus::Warning);
        assert!(HealthStatus::Warning < HealthStatus::Critical);
        assert!(HealthStatus::Critical < HealthStatus::Emergency);
    }

    #[test]
    fn test_recommendation_priority_ordering() {
        assert!(RecommendationPriority::Low < RecommendationPriority::Medium);
        assert!(RecommendationPriority::Medium < RecommendationPriority::High);
        assert!(RecommendationPriority::High < RecommendationPriority::Critical);
    }

    #[test]
    fn test_integrity_error_types() {
        let errors = vec![
            IntegrityError::HashMismatch {
                expected: "hash1".to_string(),
                actual: "hash2".to_string(),
                entity_id: "doc1".to_string(),
            },
            IntegrityError::MissingReference {
                reference_id: "ref1".to_string(),
                referenced_by: "doc1".to_string(),
            },
            IntegrityError::OrphanedData {
                entity_id: "orphan1".to_string(),
                entity_type: "chunk".to_string(),
            },
        ];

        // Ensure all error types are different
        for (i, error1) in errors.iter().enumerate() {
            for (j, error2) in errors.iter().enumerate() {
                if i != j {
                    assert_ne!(
                        std::mem::discriminant(error1),
                        std::mem::discriminant(error2)
                    );
                }
            }
        }
    }
}