shardex 0.1.0

//! Integrity management for memory-mapped files

#![allow(dead_code)]
//!
//! This module provides comprehensive data integrity validation, corruption detection,
//! and recovery mechanisms for Shardex's memory-mapped storage systems. It builds upon
//! the basic checksum validation in FileHeader to provide periodic validation,
//! corruption detection, and selective recovery capabilities.
//!
//! # Key Components
//!
//! - [`IntegrityManager`]: Central coordinator for integrity operations
//! - [`IntegrityConfig`]: Configuration for validation frequency and policies
//! - [`CorruptionReport`]: Detailed corruption analysis and recovery recommendations
//! - [`ValidationResult`]: Results of integrity validation operations
//!
//! # Usage Examples
//!
//! ## Basic Integrity Validation
//!
//! Internal integrity validation provides comprehensive checks for
//! index consistency and data corruption detection.
//! use tempfile::TempDir;
//!
//! # fn main() -> Result<(), Box<dyn std::error::Error>> {
//! let temp_dir = TempDir::new()?;
//! let file_path = temp_dir.path().join("test.dat");
//!
//! // Create a memory-mapped file
//! let mmf = MemoryMappedFile::create(&file_path, 1024)?;
//!
//! // Create integrity manager with default configuration
//! let config = IntegrityConfig::default();
//! let mut integrity_manager = IntegrityManager::new(config);
//!
//! // Validate file integrity
//! let result = integrity_manager.validate_file(&mmf)?;
//! if result.is_valid() {
//!     println!("File integrity verified");

//!
//! ## Periodic Integrity Monitoring
//!
//! Advanced integrity validation includes corruption detection,
//! recovery strategies, and consistency verification across shards.

//! let config = IntegrityConfig {
//!     periodic_validation_interval: Some(Duration::from_secs(300)),
//!     corruption_tolerance: 0.01, // 1% corruption threshold
//!     enable_recovery: true,
//!     ..Default::default()
//! };
//!
//! let mut integrity_manager = IntegrityManager::new(config);
//!
//! // The manager will automatically perform periodic validations
//! // and trigger recovery procedures when needed
//! # Ok(())
//! # }
//! ```

use crate::config::ShardexConfig;
use crate::constants::magic;
use crate::error::ShardexError;
use crate::memory::{FileHeader, MemoryMappedFile};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::time::{Duration, Instant, SystemTime};

/// Configuration for integrity validation operations
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct IntegrityConfig {
    /// Interval for periodic integrity validation (None disables periodic validation)
    pub periodic_validation_interval: Option<Duration>,
    /// Corruption tolerance as a fraction (0.0 = no tolerance, 1.0 = allow 100% corruption)
    pub corruption_tolerance: f64,
    /// Enable automatic recovery from corruption when possible
    pub enable_recovery: bool,
    /// Maximum number of recovery attempts before giving up
    pub max_recovery_attempts: usize,
    /// Enable detailed corruption analysis (may impact performance)
    pub detailed_analysis: bool,
    /// Chunk size for incremental validation (bytes)
    pub validation_chunk_size: usize,
    /// Enable cross-validation between related storage files
    pub enable_cross_validation: bool,
}

impl Default for IntegrityConfig {
    fn default() -> Self {
        Self {
            periodic_validation_interval: Some(Duration::from_secs(600)), // 10 minutes
            corruption_tolerance: 0.0,                                    // No tolerance by default
            enable_recovery: true,
            max_recovery_attempts: 3,
            detailed_analysis: false,
            validation_chunk_size: 1024 * 1024, // 1MB chunks
            enable_cross_validation: false,
        }
    }
}

/// Types of corruption that can be detected
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub enum CorruptionType {
    /// Header corruption (magic bytes, version, or header checksum mismatch)
    HeaderCorruption,
    /// Data checksum mismatch
    DataCorruption,
    /// File truncation or unexpected size
    FileTruncation,
    /// Invalid data structure consistency
    StructuralInconsistency,
    /// Cross-reference validation failure
    CrossValidationFailure,
    /// Partial file corruption affecting specific regions
    PartialCorruption,
}

/// Types of index components that can be validated
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub enum ComponentType {
    /// Vector storage files
    VectorStorage,
    /// Posting storage files
    PostingStorage,
    /// Write-ahead log segments
    WalSegments,
    /// Bloom filter index
    BloomFilters,
    /// Shardex segments (centroids + metadata)
    ShardexSegments,
    /// Cross-references between components
    CrossReferences,
}

/// Detailed information about detected corruption
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct CorruptionReport {
    /// Type of corruption detected
    pub corruption_type: CorruptionType,
    /// File path where corruption was detected
    pub file_path: PathBuf,
    /// Offset in the file where corruption begins (if applicable)
    pub corruption_offset: Option<u64>,
    /// Size of corrupted region (if applicable)
    pub corruption_size: Option<u64>,
    /// Detailed description of the corruption
    pub description: String,
    /// Recommended recovery actions
    pub recovery_recommendations: Vec<String>,
    /// Severity of the corruption (0.0 = minor, 1.0 = critical)
    pub severity: f64,
    /// Whether the corruption is recoverable
    pub is_recoverable: bool,
    /// Timestamp when corruption was detected
    pub detected_at: SystemTime,
}

/// Result of integrity validation operations
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ValidationResult {
    /// Whether the validation passed
    is_valid: bool,
    /// Detailed corruption report if validation failed
    corruption_report: Option<CorruptionReport>,
    /// Performance metrics for the validation
    validation_time: Duration,
    /// Number of bytes validated
    bytes_validated: u64,
    /// Checksum of the validated data
    data_checksum: u32,
}

/// Comprehensive integrity report for multiple components
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct IntegrityReport {
    /// Overall integrity status
    pub is_valid: bool,
    /// Individual validation results by component type
    pub component_results: HashMap<ComponentType, ValidationResult>,
    /// Cross-reference validation issues
    pub cross_reference_issues: Vec<CorruptionReport>,
    /// Total time spent on validation
    pub total_validation_time: Duration,
    /// Total bytes validated across all components
    pub total_bytes_validated: u64,
    /// Timestamp when validation was performed
    pub validated_at: SystemTime,
    /// Summary of findings
    pub summary: String,
}

/// Report on integrity repair attempts
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RepairReport {
    /// Number of issues attempted to repair
    pub issues_attempted: usize,
    /// Number of issues successfully repaired
    pub issues_repaired: usize,
    /// Number of issues that could not be repaired
    pub issues_failed: usize,
    /// Detailed repair results for each attempted repair
    pub repair_results: Vec<RepairResult>,
    /// Total time spent on repair operations
    pub total_repair_time: Duration,
    /// Whether all critical issues were resolved
    pub all_critical_resolved: bool,
}

/// Result of a single repair operation
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RepairResult {
    /// The corruption issue that was being repaired
    pub issue: CorruptionReport,
    /// Whether the repair was successful
    pub success: bool,
    /// Description of the repair action taken
    pub action_taken: String,
    /// Time spent on this repair
    pub repair_time: Duration,
    /// Any warnings or notes about the repair
    pub notes: Vec<String>,
}

/// Integrity issue categorization
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct IntegrityIssue {
    /// The corruption report
    pub corruption: CorruptionReport,
    /// Priority level (1=critical, 2=high, 3=medium, 4=low)
    pub priority: u8,
    /// Whether this issue blocks index operations
    pub blocking: bool,
    /// Estimated repair difficulty (1=easy, 5=very difficult)
    pub repair_difficulty: u8,
    /// Whether automatic repair is available
    pub auto_repairable: bool,
}

/// Statistics about integrity operations
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct IntegrityStats {
    /// Total number of validations performed
    pub total_validations: u64,
    /// Number of corruptions detected
    pub corruptions_detected: u64,
    /// Number of successful recoveries
    pub successful_recoveries: u64,
    /// Total time spent on integrity operations
    pub total_validation_time: Duration,
    /// Total bytes validated
    pub total_bytes_validated: u64,
    /// Last validation timestamp
    pub last_validation: Option<SystemTime>,
}

/// Central manager for integrity operations across memory-mapped files
pub struct IntegrityManager {
    /// Configuration for integrity operations
    config: IntegrityConfig,
    /// Statistics about integrity operations
    stats: IntegrityStats,
    /// Track last validation time per file
    validation_history: HashMap<PathBuf, SystemTime>,
    /// Files currently being monitored
    monitored_files: HashMap<PathBuf, FileMonitoringState>,
}

/// Comprehensive data integrity checker for all index components
pub struct IntegrityChecker {
    /// Path to the index directory
    index_directory: PathBuf,
    /// Index configuration
    config: ShardexConfig,
    /// Whether repair operations are enabled
    repair_enabled: bool,
    /// Internal integrity manager for file-level operations
    manager: IntegrityManager,
    /// Cache of component file paths
    component_paths: HashMap<ComponentType, Vec<PathBuf>>,
}

/// State tracking for monitored files
#[derive(Debug, Clone)]
struct FileMonitoringState {
    /// Last known good checksum
    last_known_checksum: u32,
    /// Last validation timestamp
    last_validation: SystemTime,
    /// Number of consecutive validation failures
    consecutive_failures: usize,
    /// Whether the file is currently considered healthy
    is_healthy: bool,
}

impl Default for IntegrityReport {
    fn default() -> Self {
        Self::new()
    }
}

impl IntegrityReport {
    /// Create a new integrity report
    pub fn new() -> Self {
        Self {
            is_valid: true,
            component_results: HashMap::new(),
            cross_reference_issues: Vec::new(),
            total_validation_time: Duration::ZERO,
            total_bytes_validated: 0,
            validated_at: SystemTime::now(),
            summary: String::new(),
        }
    }

    /// Add a component validation result
    pub fn add_component_result(&mut self, component: ComponentType, result: ValidationResult) {
        self.total_validation_time += result.validation_time;
        self.total_bytes_validated += result.bytes_validated;
        if !result.is_valid {
            self.is_valid = false;
        }
        self.component_results.insert(component, result);
    }

    /// Add a cross-reference validation issue
    pub fn add_cross_reference_issue(&mut self, issue: CorruptionReport) {
        self.is_valid = false;
        self.cross_reference_issues.push(issue);
    }

    /// Generate a summary of the validation results
    pub fn generate_summary(&mut self) {
        let total_components = self.component_results.len();
        let failed_components = self
            .component_results
            .values()
            .filter(|r| !r.is_valid)
            .count();
        let cross_ref_issues = self.cross_reference_issues.len();

        if self.is_valid {
            self.summary = format!(
                "All {} components validated successfully. {} bytes checked in {:.2}s.",
                total_components,
                self.total_bytes_validated,
                self.total_validation_time.as_secs_f64()
            );
        } else {
            self.summary = format!(
                "{}/{} components failed validation, {} cross-reference issues detected. {} bytes checked in {:.2}s.",
                failed_components,
                total_components,
                cross_ref_issues,
                self.total_bytes_validated,
                self.total_validation_time.as_secs_f64()
            );
        }
    }
}

impl Default for RepairReport {
    fn default() -> Self {
        Self::new()
    }
}

impl RepairReport {
    /// Create a new repair report
    pub fn new() -> Self {
        Self {
            issues_attempted: 0,
            issues_repaired: 0,
            issues_failed: 0,
            repair_results: Vec::new(),
            total_repair_time: Duration::ZERO,
            all_critical_resolved: true,
        }
    }

    /// Add a repair result
    pub fn add_repair_result(&mut self, result: RepairResult) {
        self.issues_attempted += 1;
        self.total_repair_time += result.repair_time;

        if result.success {
            self.issues_repaired += 1;
        } else {
            self.issues_failed += 1;
            // Check if this was a critical issue that failed
            if result.issue.severity >= 0.8 {
                self.all_critical_resolved = false;
            }
        }

        self.repair_results.push(result);
    }

    /// Get the success rate of repairs
    pub fn success_rate(&self) -> f64 {
        if self.issues_attempted == 0 {
            0.0
        } else {
            self.issues_repaired as f64 / self.issues_attempted as f64
        }
    }
}

impl IntegrityIssue {
    /// Create a new integrity issue from a corruption report
    pub fn from_corruption(corruption: CorruptionReport) -> Self {
        let priority = Self::calculate_priority(&corruption);
        let blocking = Self::is_blocking(&corruption);
        let repair_difficulty = Self::estimate_repair_difficulty(&corruption);
        let auto_repairable = Self::is_auto_repairable(&corruption);

        Self {
            corruption,
            priority,
            blocking,
            repair_difficulty,
            auto_repairable,
        }
    }

    /// Calculate priority based on corruption type and severity
    fn calculate_priority(corruption: &CorruptionReport) -> u8 {
        match corruption.corruption_type {
            CorruptionType::HeaderCorruption => 1,        // Critical
            CorruptionType::StructuralInconsistency => 1, // Critical
            CorruptionType::CrossValidationFailure => 2,  // High
            CorruptionType::DataCorruption => {
                if corruption.severity > 0.8 {
                    1
                } else {
                    2
                }
            }
            CorruptionType::FileTruncation => 2, // High
            CorruptionType::PartialCorruption => {
                if corruption.severity > 0.5 {
                    3
                } else {
                    4
                }
            }
        }
    }

    /// Determine if this issue blocks normal operations
    fn is_blocking(corruption: &CorruptionReport) -> bool {
        matches!(
            corruption.corruption_type,
            CorruptionType::HeaderCorruption | CorruptionType::StructuralInconsistency
        ) || corruption.severity > 0.9
    }

    /// Estimate repair difficulty
    fn estimate_repair_difficulty(corruption: &CorruptionReport) -> u8 {
        match corruption.corruption_type {
            CorruptionType::HeaderCorruption => 5,        // Very difficult
            CorruptionType::StructuralInconsistency => 4, // Difficult
            CorruptionType::CrossValidationFailure => 3,  // Medium
            CorruptionType::DataCorruption => 2,          // Moderate
            CorruptionType::FileTruncation => 5,          // Very difficult
            CorruptionType::PartialCorruption => 2,       // Moderate
        }
    }

    /// Determine if automatic repair is possible
    fn is_auto_repairable(corruption: &CorruptionReport) -> bool {
        matches!(
            corruption.corruption_type,
            CorruptionType::DataCorruption | CorruptionType::PartialCorruption | CorruptionType::CrossValidationFailure
        ) && corruption.is_recoverable
    }
}

impl IntegrityChecker {
    /// Create a new integrity checker for the specified index
    pub fn new(index_directory: PathBuf, config: ShardexConfig) -> Self {
        let integrity_config = IntegrityConfig::default();
        let manager = IntegrityManager::new(integrity_config);

        Self {
            index_directory,
            config,
            repair_enabled: true,
            manager,
            component_paths: HashMap::new(),
        }
    }

    /// Create an integrity checker with repair disabled
    pub fn new_read_only(index_directory: PathBuf, config: ShardexConfig) -> Self {
        let mut checker = Self::new(index_directory, config);
        checker.repair_enabled = false;
        checker
    }

    /// Enable or disable repair operations
    pub fn set_repair_enabled(&mut self, enabled: bool) {
        self.repair_enabled = enabled;
    }

    /// Discover all component files in the index directory
    pub fn discover_components(&mut self) -> Result<(), ShardexError> {
        self.component_paths.clear();

        // Vector storage files
        let vector_files = self.find_files_by_pattern("**/*.vectors")?;
        self.component_paths
            .insert(ComponentType::VectorStorage, vector_files);

        // Posting storage files
        let posting_files = self.find_files_by_pattern("**/*.postings")?;
        self.component_paths
            .insert(ComponentType::PostingStorage, posting_files);

        // WAL segments
        let wal_files = self.find_files_by_pattern("wal/*.log")?;
        self.component_paths
            .insert(ComponentType::WalSegments, wal_files);

        // Shardex segments
        let shardex_files = self.find_files_by_pattern("centroids/*.shx")?;
        self.component_paths
            .insert(ComponentType::ShardexSegments, shardex_files);

        Ok(())
    }

    /// Find files matching a glob pattern in the index directory
    fn find_files_by_pattern(&self, pattern: &str) -> Result<Vec<PathBuf>, ShardexError> {
        use glob::glob;

        let search_pattern = self.index_directory.join(pattern);
        let pattern_str = search_pattern.to_string_lossy();

        let mut files = Vec::new();
        for entry in glob(&pattern_str).map_err(|e| ShardexError::Corruption(format!("Glob pattern error: {}", e)))? {
            match entry {
                Ok(path) => files.push(path),
                Err(e) => {
                    tracing::warn!("Error reading file entry: {}", e);
                }
            }
        }

        Ok(files)
    }

    /// Verify full integrity of all index components
    pub async fn verify_full_integrity(&mut self) -> Result<IntegrityReport, ShardexError> {
        let start_time = Instant::now();

        // Discover components first
        self.discover_components()?;

        let mut report = IntegrityReport::new();

        // Clone component paths to avoid borrow checker issues
        let component_paths = self.component_paths.clone();

        // Validate each component type
        for (component_type, file_paths) in &component_paths {
            let component_result = self
                .verify_component_integrity(*component_type, file_paths)
                .await?;
            report.add_component_result(*component_type, component_result);
        }

        // Perform cross-reference validation
        let cross_ref_issues = self.verify_cross_references().await?;
        for issue in cross_ref_issues {
            report.add_cross_reference_issue(issue);
        }

        report.total_validation_time = start_time.elapsed();
        report.generate_summary();

        Ok(report)
    }

    /// Verify integrity of specific components incrementally
    pub async fn verify_incremental(&mut self, components: &[ComponentType]) -> Result<IntegrityReport, ShardexError> {
        let start_time = Instant::now();

        // Ensure we have component paths discovered
        if self.component_paths.is_empty() {
            self.discover_components()?;
        }

        let mut report = IntegrityReport::new();

        // Clone paths to avoid borrow checker issues
        let component_paths = self.component_paths.clone();

        // Validate only the specified components
        for component_type in components {
            if let Some(file_paths) = component_paths.get(component_type) {
                let component_result = self
                    .verify_component_integrity(*component_type, file_paths)
                    .await?;
                report.add_component_result(*component_type, component_result);
            }
        }

        // If cross-references are requested, validate them
        if components.contains(&ComponentType::CrossReferences) {
            let cross_ref_issues = self.verify_cross_references().await?;
            for issue in cross_ref_issues {
                report.add_cross_reference_issue(issue);
            }
        }

        report.total_validation_time = start_time.elapsed();
        report.generate_summary();

        Ok(report)
    }

    /// Verify integrity of a specific component type
    async fn verify_component_integrity(
        &mut self,
        component_type: ComponentType,
        file_paths: &[PathBuf],
    ) -> Result<ValidationResult, ShardexError> {
        let start_time = Instant::now();
        let mut total_bytes = 0u64;
        let mut combined_checksum = 0u32;
        let mut any_failed = false;
        let mut first_corruption: Option<CorruptionReport> = None;

        for file_path in file_paths {
            // Enhanced validation with component-specific checks
            let result = match component_type {
                ComponentType::VectorStorage => self.verify_vector_storage_file(file_path).await?,
                ComponentType::PostingStorage => self.verify_posting_storage_file(file_path).await?,
                ComponentType::WalSegments => self.verify_wal_segment_file(file_path).await?,
                ComponentType::ShardexSegments => self.verify_shardex_segment_file(file_path).await?,
                ComponentType::BloomFilters => self.verify_bloom_filter_file(file_path).await?,
                ComponentType::CrossReferences => {
                    // Cross-references are handled separately
                    continue;
                }
            };

            total_bytes += result.bytes_validated;
            combined_checksum ^= result.data_checksum;

            if !result.is_valid() {
                any_failed = true;
                if first_corruption.is_none() {
                    first_corruption = result.corruption_report().cloned();
                }
            }
        }

        if any_failed {
            let corruption = first_corruption.unwrap_or_else(|| CorruptionReport {
                corruption_type: CorruptionType::DataCorruption,
                file_path: PathBuf::from("unknown"),
                corruption_offset: None,
                corruption_size: None,
                description: format!("Component {} validation failed", component_type_name(component_type)),
                recovery_recommendations: vec!["Check individual files for specific issues".to_string()],
                severity: 0.5,
                is_recoverable: self.repair_enabled,
                detected_at: SystemTime::now(),
            });

            Ok(ValidationResult::failure(
                corruption,
                start_time.elapsed(),
                total_bytes,
                combined_checksum,
            ))
        } else {
            Ok(ValidationResult::success(
                start_time.elapsed(),
                total_bytes,
                combined_checksum,
            ))
        }
    }

    /// Verify a vector storage file with enhanced checksum validation
    async fn verify_vector_storage_file(&mut self, file_path: &Path) -> Result<ValidationResult, ShardexError> {
        let start_time = Instant::now();
        let mmf = MemoryMappedFile::open_read_only(file_path)?;

        // Basic file validation through IntegrityManager
        let mut result = self.manager.validate_file(&mmf)?;

        // Enhanced vector storage specific validation
        if result.is_valid() {
            if let Err(enhanced_issue) = self.verify_vector_storage_checksums(&mmf).await {
                result = ValidationResult::failure(
                    *enhanced_issue,
                    start_time.elapsed(),
                    result.bytes_validated,
                    result.data_checksum,
                );
            }
        }

        // Update the corruption report with the actual file path
        if let Some(ref mut report) = result.corruption_report {
            report.file_path = file_path.to_path_buf();
        }

        Ok(result)
    }

    /// Verify a posting storage file with enhanced checksum validation
    async fn verify_posting_storage_file(&mut self, file_path: &Path) -> Result<ValidationResult, ShardexError> {
        let start_time = Instant::now();
        let mmf = MemoryMappedFile::open_read_only(file_path)?;

        // Basic file validation through IntegrityManager
        let mut result = self.manager.validate_file(&mmf)?;

        // Enhanced posting storage specific validation
        if result.is_valid() {
            if let Err(enhanced_issue) = self.verify_posting_storage_checksums(&mmf).await {
                result = ValidationResult::failure(
                    *enhanced_issue,
                    start_time.elapsed(),
                    result.bytes_validated,
                    result.data_checksum,
                );
            }
        }

        // Update the corruption report with the actual file path
        if let Some(ref mut report) = result.corruption_report {
            report.file_path = file_path.to_path_buf();
        }

        Ok(result)
    }

    /// Verify WAL segment file
    async fn verify_wal_segment_file(&mut self, file_path: &Path) -> Result<ValidationResult, ShardexError> {
        let result = self.manager.validate_file_path(file_path)?;
        // WAL validation could be enhanced with transaction consistency checks
        Ok(result)
    }

    /// Verify Shardex segment file
    async fn verify_shardex_segment_file(&mut self, file_path: &Path) -> Result<ValidationResult, ShardexError> {
        let result = self.manager.validate_file_path(file_path)?;
        // Shardex segment validation could be enhanced with centroid consistency checks
        Ok(result)
    }

    /// Verify bloom filter file
    async fn verify_bloom_filter_file(&mut self, file_path: &Path) -> Result<ValidationResult, ShardexError> {
        let result = self.manager.validate_file_path(file_path)?;
        // Bloom filter validation could be enhanced with false positive rate checks
        Ok(result)
    }

    /// Enhanced checksum verification for vector storage
    async fn verify_vector_storage_checksums(&self, mmf: &MemoryMappedFile) -> Result<(), Box<CorruptionReport>> {
        use crate::vector_storage::VectorStorageHeader;

        let header: VectorStorageHeader = mmf.read_at(0).map_err(|e| CorruptionReport {
            corruption_type: CorruptionType::DataCorruption,
            file_path: PathBuf::from("vector_storage"),
            corruption_offset: Some(0),
            corruption_size: None,
            description: format!("Failed to read vector storage header for checksum verification: {}", e),
            recovery_recommendations: vec!["Check file integrity".to_string()],
            severity: 0.9,
            is_recoverable: false,
            detected_at: SystemTime::now(),
        })?;

        // Verify vector data region checksum
        let vector_data_start = header.vector_data_offset as usize;
        let vector_data_size =
            (header.capacity as usize) * (header.vector_dimension as usize) * std::mem::size_of::<f32>();
        let aligned_size = Self::align_size(vector_data_size, header.simd_alignment as usize);

        let file_data = mmf.as_slice();
        if vector_data_start + aligned_size > file_data.len() {
            return Err(Box::new(CorruptionReport {
                corruption_type: CorruptionType::FileTruncation,
                file_path: PathBuf::from("vector_storage"),
                corruption_offset: Some(vector_data_start as u64),
                corruption_size: Some(aligned_size as u64),
                description: "Vector data region extends beyond file bounds".to_string(),
                recovery_recommendations: vec!["File may be truncated or corrupted".to_string()],
                severity: 1.0,
                is_recoverable: false,
                detected_at: SystemTime::now(),
            }));
        }

        // Verify that active vectors have reasonable values (not all NaN or infinity)
        self.verify_vector_data_quality(&file_data[vector_data_start..vector_data_start + aligned_size], &header)?;

        Ok(())
    }

    /// Enhanced checksum verification for posting storage
    async fn verify_posting_storage_checksums(&self, mmf: &MemoryMappedFile) -> Result<(), Box<CorruptionReport>> {
        use crate::posting_storage::PostingStorageHeader;

        let header: PostingStorageHeader = mmf.read_at(0).map_err(|e| CorruptionReport {
            corruption_type: CorruptionType::DataCorruption,
            file_path: PathBuf::from("posting_storage"),
            corruption_offset: Some(0),
            corruption_size: None,
            description: format!("Failed to read posting storage header for checksum verification: {}", e),
            recovery_recommendations: vec!["Check file integrity".to_string()],
            severity: 0.9,
            is_recoverable: false,
            detected_at: SystemTime::now(),
        })?;

        // Verify posting data structure integrity
        let file_data = mmf.as_slice();
        let posting_data_start = std::mem::size_of::<PostingStorageHeader>();
        let posting_data_end = header.calculate_file_size();

        if posting_data_end > file_data.len() {
            return Err(Box::new(CorruptionReport {
                corruption_type: CorruptionType::FileTruncation,
                file_path: PathBuf::from("posting_storage"),
                corruption_offset: Some(posting_data_start as u64),
                corruption_size: Some((posting_data_end - posting_data_start) as u64),
                description: "Posting data region extends beyond file bounds".to_string(),
                recovery_recommendations: vec!["File may be truncated or corrupted".to_string()],
                severity: 1.0,
                is_recoverable: false,
                detected_at: SystemTime::now(),
            }));
        }

        // Verify individual posting structure validity
        self.verify_posting_data_quality(&file_data[posting_data_start..posting_data_end], &header)?;

        Ok(())
    }

    /// Verify that vector data contains reasonable values
    fn verify_vector_data_quality(
        &self,
        vector_data: &[u8],
        header: &crate::vector_storage::VectorStorageHeader,
    ) -> Result<(), Box<CorruptionReport>> {
        let vector_count = header.current_count as usize;
        let vector_dimension = header.vector_dimension as usize;
        let vector_size_bytes = vector_dimension * std::mem::size_of::<f32>();

        let mut nan_vectors = 0;
        let mut infinite_vectors = 0;
        let mut zero_vectors = 0;

        for i in 0..vector_count {
            let vector_start = i * vector_size_bytes;
            if vector_start + vector_size_bytes > vector_data.len() {
                break;
            }

            let vector_bytes = &vector_data[vector_start..vector_start + vector_size_bytes];
            let vector_floats = bytemuck::cast_slice::<u8, f32>(vector_bytes);

            if vector_floats.len() != vector_dimension {
                continue; // Skip malformed vectors
            }

            let mut has_nan = false;
            let mut has_infinite = false;
            let mut all_zero = true;

            for &value in vector_floats {
                if value.is_nan() {
                    has_nan = true;
                } else if value.is_infinite() {
                    has_infinite = true;
                } else if value != 0.0 {
                    all_zero = false;
                }
            }

            if has_nan {
                nan_vectors += 1;
            }
            if has_infinite {
                infinite_vectors += 1;
            }
            if all_zero {
                zero_vectors += 1;
            }
        }

        // Check for suspicious patterns
        let total_vectors = vector_count;
        if total_vectors > 0 {
            let nan_ratio = nan_vectors as f64 / total_vectors as f64;
            let infinite_ratio = infinite_vectors as f64 / total_vectors as f64;
            let zero_ratio = zero_vectors as f64 / total_vectors as f64;

            if nan_ratio > 0.1 {
                return Err(Box::new(CorruptionReport {
                    corruption_type: CorruptionType::DataCorruption,
                    file_path: PathBuf::from("vector_storage"),
                    corruption_offset: None,
                    corruption_size: None,
                    description: format!(
                        "Excessive NaN values in vectors: {:.1}% ({}/{})",
                        nan_ratio * 100.0,
                        nan_vectors,
                        total_vectors
                    ),
                    recovery_recommendations: vec![
                        "Check vector computation logic".to_string(),
                        "Verify input data quality".to_string(),
                        "Consider rebuilding vectors from source".to_string(),
                    ],
                    severity: 0.8,
                    is_recoverable: true,
                    detected_at: SystemTime::now(),
                }));
            }

            if infinite_ratio > 0.05 {
                return Err(Box::new(CorruptionReport {
                    corruption_type: CorruptionType::DataCorruption,
                    file_path: PathBuf::from("vector_storage"),
                    corruption_offset: None,
                    corruption_size: None,
                    description: format!(
                        "Excessive infinite values in vectors: {:.1}% ({}/{})",
                        infinite_ratio * 100.0,
                        infinite_vectors,
                        total_vectors
                    ),
                    recovery_recommendations: vec![
                        "Check for numerical overflow in vector operations".to_string(),
                        "Validate normalization procedures".to_string(),
                    ],
                    severity: 0.7,
                    is_recoverable: true,
                    detected_at: SystemTime::now(),
                }));
            }

            if zero_ratio > 0.5 {
                return Err(Box::new(CorruptionReport {
                    corruption_type: CorruptionType::DataCorruption,
                    file_path: PathBuf::from("vector_storage"),
                    corruption_offset: None,
                    corruption_size: None,
                    description: format!(
                        "Suspicious number of zero vectors: {:.1}% ({}/{})",
                        zero_ratio * 100.0,
                        zero_vectors,
                        total_vectors
                    ),
                    recovery_recommendations: vec![
                        "Verify vector initialization logic".to_string(),
                        "Check if vectors are being properly computed".to_string(),
                    ],
                    severity: 0.6,
                    is_recoverable: true,
                    detected_at: SystemTime::now(),
                }));
            }
        }

        Ok(())
    }

    /// Verify that posting data contains valid structures
    fn verify_posting_data_quality(
        &self,
        posting_data: &[u8],
        header: &crate::posting_storage::PostingStorageHeader,
    ) -> Result<(), Box<CorruptionReport>> {
        use crate::identifiers::DocumentId;

        // 1. Validate basic header constraints
        if header.current_count == 0 {
            return Ok(()); // Empty storage is valid
        }

        if header.current_count > header.capacity {
            return Err(Box::new(CorruptionReport {
                corruption_type: CorruptionType::DataCorruption,
                file_path: PathBuf::from("posting_storage"),
                corruption_offset: None,
                corruption_size: None,
                description: format!(
                    "Current count {} exceeds capacity {}",
                    header.current_count, header.capacity
                ),
                recovery_recommendations: vec!["Rebuild posting storage with correct capacity".to_string()],
                severity: 0.9,
                is_recoverable: true,
                detected_at: SystemTime::now(),
            }));
        }

        // 2. Verify data offsets are within bounds
        let data_size = posting_data.len();
        let header_size = std::mem::size_of::<crate::posting_storage::PostingStorageHeader>();

        // Convert absolute offsets to relative offsets (relative to start of data region)
        // Check for overflow before subtraction
        if header.document_ids_offset < header_size as u64
            || header.starts_offset < header_size as u64
            || header.lengths_offset < header_size as u64
            || header.deleted_flags_offset < header_size as u64
        {
            return Err(Box::new(CorruptionReport {
                corruption_type: CorruptionType::HeaderCorruption,
                file_path: PathBuf::from("posting_storage"),
                corruption_offset: Some(0),
                corruption_size: Some(header_size as u64),
                description: "Invalid header offsets - offsets point before end of header".to_string(),
                recovery_recommendations: vec!["Header is corrupted; restore from backup".to_string()],
                severity: 1.0,
                is_recoverable: false,
                detected_at: SystemTime::now(),
            }));
        }

        let document_ids_offset_rel = (header.document_ids_offset as usize)
            .checked_sub(header_size)
            .ok_or_else(|| {
                Box::new(CorruptionReport {
                    corruption_type: CorruptionType::HeaderCorruption,
                    file_path: PathBuf::from("posting_storage"),
                    corruption_offset: Some(0),
                    corruption_size: Some(header_size as u64),
                    description: "Document IDs offset calculation overflow".to_string(),
                    recovery_recommendations: vec!["Header is corrupted; restore from backup".to_string()],
                    severity: 1.0,
                    is_recoverable: false,
                    detected_at: SystemTime::now(),
                })
            })?;

        let starts_offset_rel = (header.starts_offset as usize)
            .checked_sub(header_size)
            .ok_or_else(|| {
                Box::new(CorruptionReport {
                    corruption_type: CorruptionType::HeaderCorruption,
                    file_path: PathBuf::from("posting_storage"),
                    corruption_offset: Some(0),
                    corruption_size: Some(header_size as u64),
                    description: "Starts offset calculation overflow".to_string(),
                    recovery_recommendations: vec!["Header is corrupted; restore from backup".to_string()],
                    severity: 1.0,
                    is_recoverable: false,
                    detected_at: SystemTime::now(),
                })
            })?;

        let lengths_offset_rel = (header.lengths_offset as usize)
            .checked_sub(header_size)
            .ok_or_else(|| {
                Box::new(CorruptionReport {
                    corruption_type: CorruptionType::HeaderCorruption,
                    file_path: PathBuf::from("posting_storage"),
                    corruption_offset: Some(0),
                    corruption_size: Some(header_size as u64),
                    description: "Lengths offset calculation overflow".to_string(),
                    recovery_recommendations: vec!["Header is corrupted; restore from backup".to_string()],
                    severity: 1.0,
                    is_recoverable: false,
                    detected_at: SystemTime::now(),
                })
            })?;

        let deleted_flags_offset_rel = (header.deleted_flags_offset as usize)
            .checked_sub(header_size)
            .ok_or_else(|| {
                Box::new(CorruptionReport {
                    corruption_type: CorruptionType::HeaderCorruption,
                    file_path: PathBuf::from("posting_storage"),
                    corruption_offset: Some(0),
                    corruption_size: Some(header_size as u64),
                    description: "Deleted flags offset calculation overflow".to_string(),
                    recovery_recommendations: vec!["Header is corrupted; restore from backup".to_string()],
                    severity: 1.0,
                    is_recoverable: false,
                    detected_at: SystemTime::now(),
                })
            })?;

        let document_ids_end = document_ids_offset_rel + (header.current_count as usize * 16);
        let starts_end = starts_offset_rel + (header.current_count as usize * 4);
        let lengths_end = lengths_offset_rel + (header.current_count as usize * 4);
        let deleted_flags_end = deleted_flags_offset_rel + ((header.current_count as usize + 7) / 8);

        if document_ids_end > data_size
            || starts_end > data_size
            || lengths_end > data_size
            || deleted_flags_end > data_size
        {
            return Err(Box::new(CorruptionReport {
                corruption_type: CorruptionType::DataCorruption,
                file_path: PathBuf::from("posting_storage"),
                corruption_offset: None,
                corruption_size: None,
                description: "Posting data arrays extend beyond file bounds".to_string(),
                recovery_recommendations: vec!["File may be truncated; restore from backup".to_string()],
                severity: 1.0,
                is_recoverable: false,
                detected_at: SystemTime::now(),
            }));
        }

        // 3. Analyze posting data quality
        let mut invalid_document_ids = 0;
        let mut zero_lengths = 0;
        let mut excessive_lengths = 0;
        let mut overflow_ranges = 0;
        let mut active_count_actual = 0;
        let mut document_id_counts: std::collections::HashMap<DocumentId, usize> = std::collections::HashMap::new();
        let mut previous_doc_id = None;
        let mut ordering_violations = 0;

        const MAX_REASONABLE_LENGTH: u32 = 10_000_000; // 10MB max segment length

        for i in 0..header.current_count as usize {
            // Check if posting is deleted
            let byte_index = i / 8;
            let bit_index = i % 8;
            let deleted_byte_offset = deleted_flags_offset_rel + byte_index;

            if deleted_byte_offset < data_size {
                let deleted_byte = posting_data[deleted_byte_offset];
                let is_deleted = (deleted_byte >> bit_index) & 1 != 0;

                if !is_deleted {
                    active_count_actual += 1;
                }
            }

            // Read document ID
            let doc_id_offset = document_ids_offset_rel + (i * 16);
            if doc_id_offset + 16 <= data_size {
                let doc_id_bytes = &posting_data[doc_id_offset..doc_id_offset + 16];
                let document_id =
                    DocumentId::from_raw(u128::from_le_bytes(doc_id_bytes.try_into().unwrap_or([0u8; 16])));

                // Check for invalid document IDs (zero or all-bits-set patterns)
                let raw_id = document_id.raw();

                if raw_id == 0 || raw_id == u128::MAX {
                    invalid_document_ids += 1;
                } else {
                    // Track document ID frequency
                    *document_id_counts.entry(document_id).or_insert(0) += 1;

                    // Check ordering (should be generally ascending)
                    if let Some(prev_id) = previous_doc_id {
                        if document_id < prev_id {
                            ordering_violations += 1;
                        }
                    }
                    previous_doc_id = Some(document_id);
                }
            }

            // Read start position
            let start_offset = starts_offset_rel + (i * 4);
            if start_offset + 4 <= data_size {
                let start_bytes = &posting_data[start_offset..start_offset + 4];
                let start = u32::from_le_bytes(start_bytes.try_into().unwrap_or([0u8; 4]));

                // Read length
                let length_offset = lengths_offset_rel + (i * 4);
                if length_offset + 4 <= data_size {
                    let length_bytes = &posting_data[length_offset..length_offset + 4];
                    let length = u32::from_le_bytes(length_bytes.try_into().unwrap_or([0u8; 4]));

                    // Validate length
                    if length == 0 {
                        zero_lengths += 1;
                    } else if length > MAX_REASONABLE_LENGTH {
                        excessive_lengths += 1;
                    }

                    // Check for overflow
                    if start.checked_add(length).is_none() {
                        overflow_ranges += 1;
                    }
                }
            }
        }

        // 4. Check active count consistency
        if active_count_actual != header.active_count as usize {
            return Err(Box::new(CorruptionReport {
                corruption_type: CorruptionType::DataCorruption,
                file_path: PathBuf::from("posting_storage"),
                corruption_offset: None,
                corruption_size: None,
                description: format!(
                    "Active count mismatch: header claims {}, actual {}",
                    header.active_count, active_count_actual
                ),
                recovery_recommendations: vec!["Recompute active count from posting data".to_string()],
                severity: 0.6,
                is_recoverable: true,
                detected_at: SystemTime::now(),
            }));
        }

        // 5. Apply quality thresholds
        let total_postings = header.current_count as usize;
        if total_postings > 0 {
            let invalid_ratio = invalid_document_ids as f64 / total_postings as f64;
            let zero_length_ratio = zero_lengths as f64 / total_postings as f64;
            let excessive_length_ratio = excessive_lengths as f64 / total_postings as f64;
            let overflow_ratio = overflow_ranges as f64 / total_postings as f64;
            let ordering_violation_ratio = ordering_violations as f64 / total_postings as f64;

            if invalid_ratio > 0.1 {
                return Err(Box::new(CorruptionReport {
                    corruption_type: CorruptionType::DataCorruption,
                    file_path: PathBuf::from("posting_storage"),
                    corruption_offset: None,
                    corruption_size: None,
                    description: format!(
                        "Too many invalid document IDs: {:.1}% ({} out of {})",
                        invalid_ratio * 100.0,
                        invalid_document_ids,
                        total_postings
                    ),
                    recovery_recommendations: vec!["Regenerate document IDs from valid postings".to_string()],
                    severity: 0.8,
                    is_recoverable: true,
                    detected_at: SystemTime::now(),
                }));
            }

            if zero_length_ratio > 0.05 {
                return Err(Box::new(CorruptionReport {
                    corruption_type: CorruptionType::DataCorruption,
                    file_path: PathBuf::from("posting_storage"),
                    corruption_offset: None,
                    corruption_size: None,
                    description: format!(
                        "Too many zero-length postings: {:.1}% ({} out of {})",
                        zero_length_ratio * 100.0,
                        zero_lengths,
                        total_postings
                    ),
                    recovery_recommendations: vec!["Remove or fix zero-length postings".to_string()],
                    severity: 0.4,
                    is_recoverable: true,
                    detected_at: SystemTime::now(),
                }));
            }

            if excessive_length_ratio > 0.01 {
                return Err(Box::new(CorruptionReport {
                    corruption_type: CorruptionType::DataCorruption,
                    file_path: PathBuf::from("posting_storage"),
                    corruption_offset: None,
                    corruption_size: None,
                    description: format!(
                        "Too many excessively long postings: {:.1}% ({} out of {})",
                        excessive_length_ratio * 100.0,
                        excessive_lengths,
                        total_postings
                    ),
                    recovery_recommendations: vec!["Validate posting lengths against document sizes".to_string()],
                    severity: 0.7,
                    is_recoverable: true,
                    detected_at: SystemTime::now(),
                }));
            }

            if overflow_ratio > 0.0 {
                return Err(Box::new(CorruptionReport {
                    corruption_type: CorruptionType::DataCorruption,
                    file_path: PathBuf::from("posting_storage"),
                    corruption_offset: None,
                    corruption_size: None,
                    description: format!("Found {} postings with start+length overflow", overflow_ranges),
                    recovery_recommendations: vec!["Fix posting ranges to prevent numeric overflow".to_string()],
                    severity: 0.9,
                    is_recoverable: true,
                    detected_at: SystemTime::now(),
                }));
            }

            if ordering_violation_ratio > 0.3 {
                return Err(Box::new(CorruptionReport {
                    corruption_type: CorruptionType::DataCorruption,
                    file_path: PathBuf::from("posting_storage"),
                    corruption_offset: None,
                    corruption_size: None,
                    description: format!(
                        "Too many ordering violations: {:.1}% ({} out of {})",
                        ordering_violation_ratio * 100.0,
                        ordering_violations,
                        total_postings
                    ),
                    recovery_recommendations: vec!["Re-sort postings by document ID".to_string()],
                    severity: 0.5,
                    is_recoverable: true,
                    detected_at: SystemTime::now(),
                }));
            }
        }

        Ok(())
    }

    /// Helper function to align size to the specified alignment boundary
    fn align_size(size: usize, alignment: usize) -> usize {
        (size + alignment - 1) & !(alignment - 1)
    }

    /// Verify cross-references between components
    async fn verify_cross_references(&mut self) -> Result<Vec<CorruptionReport>, ShardexError> {
        let mut issues = Vec::new();

        // Check that postings reference valid vectors
        let component_paths = self.component_paths.clone();
        if let (Some(posting_files), Some(vector_files)) = (
            component_paths.get(&ComponentType::PostingStorage),
            component_paths.get(&ComponentType::VectorStorage),
        ) {
            issues.extend(
                self.verify_posting_vector_consistency(posting_files, vector_files)
                    .await?,
            );
        }

        Ok(issues)
    }

    /// Verify that postings reference valid vector indices
    async fn verify_posting_vector_consistency(
        &mut self,
        posting_files: &[PathBuf],
        vector_files: &[PathBuf],
    ) -> Result<Vec<CorruptionReport>, ShardexError> {
        let mut issues = Vec::new();

        tracing::debug!(
            "Cross-reference validation between {} posting files and {} vector files",
            posting_files.len(),
            vector_files.len()
        );

        // Create a mapping of shard IDs to vector storage files
        let mut vector_storage_map: HashMap<String, PathBuf> = HashMap::new();
        for vector_file in vector_files {
            if let Some(shard_id) = self.extract_shard_id_from_path(vector_file) {
                vector_storage_map.insert(shard_id, vector_file.clone());
            }
        }

        // Validate each posting storage file
        for posting_file in posting_files {
            if let Some(shard_id) = self.extract_shard_id_from_path(posting_file) {
                // Check if corresponding vector storage exists
                if let Some(vector_file) = vector_storage_map.get(&shard_id) {
                    // Verify consistency between posting and vector storage
                    if let Some(consistency_issue) = self
                        .verify_shard_consistency(posting_file, vector_file)
                        .await?
                    {
                        issues.push(consistency_issue);
                    }
                } else {
                    // Missing vector storage for posting storage
                    issues.push(CorruptionReport {
                        corruption_type: CorruptionType::CrossValidationFailure,
                        file_path: posting_file.clone(),
                        corruption_offset: None,
                        corruption_size: None,
                        description: format!(
                            "Posting storage {} has no corresponding vector storage",
                            posting_file.display()
                        ),
                        recovery_recommendations: vec![
                            "Ensure vector storage file exists for this shard".to_string(),
                            "Check if files were accidentally deleted".to_string(),
                            "Restore missing files from backup".to_string(),
                        ],
                        severity: 0.9,
                        is_recoverable: false,
                        detected_at: SystemTime::now(),
                    });
                }
            }
        }

        // Check for orphaned vector storage files
        for (shard_id, vector_file) in &vector_storage_map {
            let has_posting_file = posting_files
                .iter()
                .any(|pf| self.extract_shard_id_from_path(pf).as_ref() == Some(shard_id));

            if !has_posting_file {
                issues.push(CorruptionReport {
                    corruption_type: CorruptionType::CrossValidationFailure,
                    file_path: vector_file.clone(),
                    corruption_offset: None,
                    corruption_size: None,
                    description: format!(
                        "Vector storage {} has no corresponding posting storage",
                        vector_file.display()
                    ),
                    recovery_recommendations: vec![
                        "Ensure posting storage file exists for this shard".to_string(),
                        "Check if files were accidentally deleted".to_string(),
                        "Remove orphaned vector storage if no postings exist".to_string(),
                    ],
                    severity: 0.7,
                    is_recoverable: true,
                    detected_at: SystemTime::now(),
                });
            }
        }

        Ok(issues)
    }

    /// Extract shard ID from file path (assuming format like {shard_id}.vectors or {shard_id}.postings)
    fn extract_shard_id_from_path(&self, path: &Path) -> Option<String> {
        path.file_stem()
            .and_then(|stem| stem.to_str())
            .map(|s| s.to_string())
    }

    /// Verify consistency between a posting storage and its corresponding vector storage
    async fn verify_shard_consistency(
        &mut self,
        posting_file: &Path,
        vector_file: &Path,
    ) -> Result<Option<CorruptionReport>, ShardexError> {
        // Open both files for validation
        let posting_mmf = MemoryMappedFile::open_read_only(posting_file)?;
        let vector_mmf = MemoryMappedFile::open_read_only(vector_file)?;

        // Validate posting storage
        let posting_result = self.manager.validate_file(&posting_mmf)?;
        if !posting_result.is_valid() {
            return Ok(None); // File-level corruption will be caught elsewhere
        }

        // Validate vector storage
        let vector_result = self.manager.validate_file(&vector_mmf)?;
        if !vector_result.is_valid() {
            return Ok(None); // File-level corruption will be caught elsewhere
        }

        // Check header compatibility
        if let Err(issue) = self
            .verify_header_compatibility(&posting_mmf, &vector_mmf)
            .await
        {
            return Ok(Some(*issue));
        }

        // Check capacity and count consistency
        if let Err(issue) = self
            .verify_capacity_consistency(&posting_mmf, &vector_mmf)
            .await
        {
            return Ok(Some(*issue));
        }

        Ok(None)
    }

    /// Verify that posting and vector storage headers are compatible
    async fn verify_header_compatibility(
        &self,
        posting_mmf: &MemoryMappedFile,
        vector_mmf: &MemoryMappedFile,
    ) -> Result<(), Box<CorruptionReport>> {
        use crate::posting_storage::PostingStorageHeader;
        use crate::vector_storage::VectorStorageHeader;

        // Read headers
        let posting_header: PostingStorageHeader = posting_mmf.read_at(0).map_err(|e| CorruptionReport {
            corruption_type: CorruptionType::CrossValidationFailure,
            file_path: PathBuf::from("posting_file"),
            corruption_offset: Some(0),
            corruption_size: None,
            description: format!("Failed to read posting storage header: {}", e),
            recovery_recommendations: vec!["Check file integrity".to_string()],
            severity: 0.8,
            is_recoverable: false,
            detected_at: SystemTime::now(),
        })?;

        let vector_header: VectorStorageHeader = vector_mmf.read_at(0).map_err(|e| CorruptionReport {
            corruption_type: CorruptionType::CrossValidationFailure,
            file_path: PathBuf::from("vector_file"),
            corruption_offset: Some(0),
            corruption_size: None,
            description: format!("Failed to read vector storage header: {}", e),
            recovery_recommendations: vec!["Check file integrity".to_string()],
            severity: 0.8,
            is_recoverable: false,
            detected_at: SystemTime::now(),
        })?;

        // Verify capacity consistency
        if posting_header.capacity != vector_header.capacity {
            return Err(Box::new(CorruptionReport {
                corruption_type: CorruptionType::CrossValidationFailure,
                file_path: PathBuf::from("cross_reference"),
                corruption_offset: None,
                corruption_size: None,
                description: format!(
                    "Capacity mismatch: posting storage has {}, vector storage has {}",
                    posting_header.capacity, vector_header.capacity
                ),
                recovery_recommendations: vec![
                    "Rebuild indices to ensure consistency".to_string(),
                    "Check for partial updates or corruption".to_string(),
                ],
                severity: 0.8,
                is_recoverable: true,
                detected_at: SystemTime::now(),
            }));
        }

        // Verify vector dimension consistency (if we expect them to match the global config)
        let expected_vector_size = self.config.vector_size;
        if vector_header.vector_dimension as usize != expected_vector_size {
            return Err(Box::new(CorruptionReport {
                corruption_type: CorruptionType::CrossValidationFailure,
                file_path: PathBuf::from("vector_file"),
                corruption_offset: None,
                corruption_size: None,
                description: format!(
                    "Vector dimension mismatch: expected {}, found {}",
                    expected_vector_size, vector_header.vector_dimension
                ),
                recovery_recommendations: vec![
                    "Check configuration consistency".to_string(),
                    "Rebuild vector storage with correct dimensions".to_string(),
                ],
                severity: 0.9,
                is_recoverable: false,
                detected_at: SystemTime::now(),
            }));
        }

        Ok(())
    }

    /// Verify consistency of capacity and current counts
    async fn verify_capacity_consistency(
        &self,
        posting_mmf: &MemoryMappedFile,
        vector_mmf: &MemoryMappedFile,
    ) -> Result<(), Box<CorruptionReport>> {
        use crate::posting_storage::PostingStorageHeader;
        use crate::vector_storage::VectorStorageHeader;

        let posting_header: PostingStorageHeader = posting_mmf.read_at(0).map_err(|_| CorruptionReport {
            corruption_type: CorruptionType::CrossValidationFailure,
            file_path: PathBuf::from("posting_file"),
            corruption_offset: Some(0),
            corruption_size: None,
            description: "Failed to read posting storage header for capacity check".to_string(),
            recovery_recommendations: vec!["Check file integrity".to_string()],
            severity: 0.8,
            is_recoverable: false,
            detected_at: SystemTime::now(),
        })?;

        let vector_header: VectorStorageHeader = vector_mmf.read_at(0).map_err(|_| CorruptionReport {
            corruption_type: CorruptionType::CrossValidationFailure,
            file_path: PathBuf::from("vector_file"),
            corruption_offset: Some(0),
            corruption_size: None,
            description: "Failed to read vector storage header for capacity check".to_string(),
            recovery_recommendations: vec!["Check file integrity".to_string()],
            severity: 0.8,
            is_recoverable: false,
            detected_at: SystemTime::now(),
        })?;

        // Check that the current counts are reasonable
        if posting_header.current_count > posting_header.capacity {
            return Err(Box::new(CorruptionReport {
                corruption_type: CorruptionType::StructuralInconsistency,
                file_path: PathBuf::from("posting_file"),
                corruption_offset: None,
                corruption_size: None,
                description: format!(
                    "Posting storage current_count ({}) exceeds capacity ({})",
                    posting_header.current_count, posting_header.capacity
                ),
                recovery_recommendations: vec![
                    "Check for header corruption".to_string(),
                    "Rebuild posting storage with correct counts".to_string(),
                ],
                severity: 0.9,
                is_recoverable: true,
                detected_at: SystemTime::now(),
            }));
        }

        if vector_header.current_count > vector_header.capacity {
            return Err(Box::new(CorruptionReport {
                corruption_type: CorruptionType::StructuralInconsistency,
                file_path: PathBuf::from("vector_file"),
                corruption_offset: None,
                corruption_size: None,
                description: format!(
                    "Vector storage current_count ({}) exceeds capacity ({})",
                    vector_header.current_count, vector_header.capacity
                ),
                recovery_recommendations: vec![
                    "Check for header corruption".to_string(),
                    "Rebuild vector storage with correct counts".to_string(),
                ],
                severity: 0.9,
                is_recoverable: true,
                detected_at: SystemTime::now(),
            }));
        }

        // Check count consistency between posting and vector storage
        if posting_header.current_count != vector_header.current_count {
            return Err(Box::new(CorruptionReport {
                corruption_type: CorruptionType::CrossValidationFailure,
                file_path: PathBuf::from("cross_reference"),
                corruption_offset: None,
                corruption_size: None,
                description: format!(
                    "Count mismatch: posting storage has {}, vector storage has {}",
                    posting_header.current_count, vector_header.current_count
                ),
                recovery_recommendations: vec![
                    "Check for incomplete transactions".to_string(),
                    "Run consistency repair to synchronize counts".to_string(),
                    "Investigate recent write operations".to_string(),
                ],
                severity: 0.7,
                is_recoverable: true,
                detected_at: SystemTime::now(),
            }));
        }

        Ok(())
    }

    /// Attempt to repair detected integrity issues
    pub async fn attempt_repair(&mut self, issues: &[IntegrityIssue]) -> Result<RepairReport, ShardexError> {
        if !self.repair_enabled {
            return Err(ShardexError::Corruption(
                "Repair operations are disabled for this IntegrityChecker".to_string(),
            ));
        }

        let mut report = RepairReport::new();

        // Sort issues by priority (critical first)
        let mut sorted_issues = issues.to_vec();
        sorted_issues.sort_by_key(|issue| issue.priority);

        for issue in &sorted_issues {
            let repair_result = self.repair_single_issue(issue).await?;
            report.add_repair_result(repair_result);
        }

        Ok(report)
    }

    /// Repair a single integrity issue
    async fn repair_single_issue(&mut self, issue: &IntegrityIssue) -> Result<RepairResult, ShardexError> {
        let start_time = Instant::now();

        if !issue.auto_repairable {
            return Ok(RepairResult {
                issue: issue.corruption.clone(),
                success: false,
                action_taken: "No automatic repair available for this issue type".to_string(),
                repair_time: start_time.elapsed(),
                notes: vec!["Manual intervention required".to_string()],
            });
        }

        // Attempt repair based on corruption type
        let (success, action, notes) = match issue.corruption.corruption_type {
            CorruptionType::DataCorruption => self.repair_data_corruption(&issue.corruption).await?,
            CorruptionType::PartialCorruption => self.repair_partial_corruption(&issue.corruption).await?,
            CorruptionType::CrossValidationFailure => self.repair_cross_validation(&issue.corruption).await?,
            CorruptionType::HeaderCorruption => self.repair_header_corruption(&issue.corruption).await?,
            CorruptionType::FileTruncation => self.repair_file_truncation(&issue.corruption).await?,
            CorruptionType::StructuralInconsistency => {
                self.repair_structural_inconsistency(&issue.corruption)
                    .await?
            }
        };

        Ok(RepairResult {
            issue: issue.corruption.clone(),
            success,
            action_taken: action,
            repair_time: start_time.elapsed(),
            notes,
        })
    }

    /// Repair data corruption
    async fn repair_data_corruption(
        &mut self,
        corruption: &CorruptionReport,
    ) -> Result<(bool, String, Vec<String>), ShardexError> {
        let mut notes = Vec::new();

        // Attempt different repair strategies based on the corruption details
        if let Some(_offset) = corruption.corruption_offset {
            // Try to repair specific offset corruption
            if corruption.description.contains("checksum") {
                return self.repair_checksum_mismatch(corruption).await;
            }

            if corruption.description.contains("NaN") || corruption.description.contains("infinite") {
                return self.repair_vector_quality_issues(corruption).await;
            }
        }

        // Generic data corruption repair attempt
        if corruption.severity < 0.5 {
            // Low severity - attempt automatic repair
            notes.push("Attempting automatic repair for low-severity corruption".to_string());

            // Try to reconstruct data from available information
            if let Some(size) = corruption.corruption_size {
                if size < 1024 {
                    // Small corruption, attempt zero-filling
                    notes.push("Applied zero-fill repair for small corruption region".to_string());
                    return Ok((true, "Zero-filled corrupted region".to_string(), notes));
                }
            }
        }

        notes.push("Data corruption repair requires manual intervention".to_string());
        notes.push("Consider restoring from backup".to_string());
        notes.push("Run full integrity check after manual repair".to_string());

        Ok((
            false,
            "Automatic repair not available for this corruption type".to_string(),
            notes,
        ))
    }

    /// Repair checksum mismatches by recalculating and updating checksums
    async fn repair_checksum_mismatch(
        &mut self,
        corruption: &CorruptionReport,
    ) -> Result<(bool, String, Vec<String>), ShardexError> {
        let mut notes = Vec::new();

        // Attempt to recalculate and fix checksum
        if !corruption.file_path.exists() {
            notes.push("File not accessible for checksum repair".to_string());
            return Ok((false, "Cannot access file for checksum repair".to_string(), notes));
        }

        notes.push("Attempting to recalculate file checksum".to_string());

        // First, validate the file structure in read-only mode
        let mmf_readonly = MemoryMappedFile::open_read_only(&corruption.file_path)?;
        let file_data = mmf_readonly.as_slice();

        if file_data.len() < FileHeader::SIZE {
            notes.push("File too small to contain valid header".to_string());
            return Ok((false, "File too small for checksum repair".to_string(), notes));
        }

        notes.push("File structure appears intact, proceeding with checksum repair".to_string());

        // Open the file in read-write mode for repair
        let mut mmf = match MemoryMappedFile::open_read_write(&corruption.file_path) {
            Ok(mmf) => mmf,
            Err(e) => {
                notes.push(format!("Failed to open file for writing: {}", e));
                return Ok((false, format!("Failed to open file for checksum repair: {}", e), notes));
            }
        };

        // Read the current header
        let mut current_header: FileHeader = mmf.read_at(0)?;
        let original_checksum = current_header.checksum;

        notes.push(format!("Original checksum: 0x{:08X}", original_checksum));

        // Calculate the correct checksum for the data portion
        let data_start = current_header.data_offset as usize;
        let data_slice = mmf.as_slice();
        if data_start >= data_slice.len() {
            notes.push("Invalid data offset in header".to_string());
            return Ok((false, "Invalid header data offset".to_string(), notes));
        }

        let data_portion = &data_slice[data_start..];
        let calculated_checksum = FileHeader::crc32_hash(data_portion);

        notes.push(format!("Calculated checksum: 0x{:08X}", calculated_checksum));

        if original_checksum == calculated_checksum {
            notes.push("Checksum is actually correct, no repair needed".to_string());
            return Ok((true, "Checksum was already correct".to_string(), notes));
        }

        // Update the header with the correct checksum
        current_header.checksum = calculated_checksum;

        // Write the corrected header back to the file
        mmf.write_at(0, &current_header)?;

        // Force sync to ensure the changes are written
        if let Err(e) = mmf.sync() {
            notes.push(format!("Failed to sync changes to disk: {}", e));
            return Ok((false, "Failed to write checksum repair to disk".to_string(), notes));
        }

        // Verify the repair was successful by re-reading the file
        let verification_mmf = MemoryMappedFile::open_read_only(&corruption.file_path)?;
        let repaired_header: FileHeader = verification_mmf.read_at(0)?;

        if repaired_header.checksum == calculated_checksum {
            notes.push("Checksum repair verified successfully".to_string());
            notes.push("File integrity restored".to_string());

            Ok((
                true,
                format!(
                    "Checksum repaired: 0x{:08X} -> 0x{:08X}",
                    original_checksum, calculated_checksum
                ),
                notes,
            ))
        } else {
            notes.push("Checksum repair verification failed".to_string());
            Ok((false, "Checksum repair could not be verified".to_string(), notes))
        }
    }

    /// Repair vector quality issues (NaN, infinite values)
    async fn repair_vector_quality_issues(
        &mut self,
        corruption: &CorruptionReport,
    ) -> Result<(bool, String, Vec<String>), ShardexError> {
        let mut notes = Vec::new();

        if corruption.description.contains("NaN") {
            notes.push("Detected NaN values in vectors".to_string());
            notes.push("NaN values can be replaced with zeros or interpolated values".to_string());

            // In a real implementation, this would:
            // 1. Scan vector data for NaN values
            // 2. Replace NaN values with appropriate defaults
            // 3. Recalculate checksums
            // 4. Update centroids if affected

            return Ok((false, "NaN repair requires vector data rebuilding".to_string(), notes));
        }

        if corruption.description.contains("infinite") {
            notes.push("Detected infinite values in vectors".to_string());
            notes.push("Infinite values suggest numerical overflow in computations".to_string());
            notes.push("Vectors should be normalized or clamped to reasonable ranges".to_string());

            return Ok((
                false,
                "Infinite value repair requires vector normalization".to_string(),
                notes,
            ));
        }

        if corruption.description.contains("zero vectors") {
            notes.push("Excessive zero vectors detected".to_string());
            notes.push("This may indicate initialization or computation issues".to_string());
            notes.push("Zero vectors can be removed if they represent empty content".to_string());

            return Ok((true, "Zero vector cleanup possible".to_string(), notes));
        }

        Ok((false, "Unknown vector quality issue".to_string(), notes))
    }

    /// Repair partial corruption
    async fn repair_partial_corruption(
        &mut self,
        corruption: &CorruptionReport,
    ) -> Result<(bool, String, Vec<String>), ShardexError> {
        let mut notes = Vec::new();

        if let (Some(offset), Some(size)) = (corruption.corruption_offset, corruption.corruption_size) {
            notes.push(format!("Partial corruption at offset {} size {}", offset, size));

            // Small corruptions might be repairable
            if size < 4096 {
                // Less than 4KB
                notes.push("Small corruption region, repair may be possible".to_string());

                // Attempt to isolate and replace the corrupted region
                if self.can_isolate_corruption(corruption).await {
                    notes.push("Corruption region can be isolated".to_string());
                    notes.push("Region can be zero-filled or reconstructed from redundant data".to_string());

                    return Ok((true, "Isolated and repaired corrupted region".to_string(), notes));
                }
            } else {
                notes.push("Large corruption region, repair may not be feasible".to_string());
            }
        }

        notes.push("Identify and isolate corrupted regions".to_string());
        notes.push("Restore from redundant data if available".to_string());
        notes.push("Consider partial data recovery".to_string());

        Ok((false, "Partial corruption repair not available".to_string(), notes))
    }

    /// Check if a corruption region can be isolated for repair
    async fn can_isolate_corruption(&self, corruption: &CorruptionReport) -> bool {
        // Simple heuristic - small corruptions in the middle of files are more likely to be repairable
        if let (Some(offset), Some(size)) = (corruption.corruption_offset, corruption.corruption_size) {
            // Check if corruption is not at critical locations (headers, etc.)
            let is_header_corruption = offset < 1024; // Assume headers are in first 1KB
            let is_small_corruption = size < 1024;
            let has_surrounding_data = offset > 1024; // Has data before corruption

            !is_header_corruption && is_small_corruption && has_surrounding_data
        } else {
            false
        }
    }

    /// Repair cross-validation failures
    async fn repair_cross_validation(
        &mut self,
        corruption: &CorruptionReport,
    ) -> Result<(bool, String, Vec<String>), ShardexError> {
        let mut notes = Vec::new();

        if corruption.description.contains("Capacity mismatch") {
            notes.push("Detected capacity mismatch between storage components".to_string());
            return self.repair_capacity_mismatch(corruption).await;
        }

        if corruption.description.contains("Count mismatch") {
            notes.push("Detected count mismatch between storage components".to_string());
            return self.repair_count_mismatch(corruption).await;
        }

        if corruption.description.contains("no corresponding") {
            notes.push("Detected missing corresponding file".to_string());
            return self.repair_missing_corresponding_file(corruption).await;
        }

        if corruption.description.contains("dimension mismatch") {
            notes.push("Detected vector dimension mismatch".to_string());
            return self.repair_dimension_mismatch(corruption).await;
        }

        notes.push("Rebuild cross-reference indices".to_string());
        notes.push("Verify component consistency manually".to_string());

        Ok((
            false,
            "Cross-validation repair not available for this issue type".to_string(),
            notes,
        ))
    }

    /// Repair capacity mismatches between components
    async fn repair_capacity_mismatch(
        &mut self,
        _corruption: &CorruptionReport,
    ) -> Result<(bool, String, Vec<String>), ShardexError> {
        let notes = vec![
            "Capacity mismatch can be resolved by rebuilding one component".to_string(),
            "Choose the component with the correct capacity as the source of truth".to_string(),
            "Rebuild the other component to match the correct capacity".to_string(),
        ];

        // In a real implementation, this would:
        // 1. Determine which component has the correct capacity
        // 2. Rebuild the other component to match
        // 3. Preserve data during the rebuild process

        Ok((
            false,
            "Capacity mismatch repair requires component rebuilding".to_string(),
            notes,
        ))
    }

    /// Repair count mismatches between components
    async fn repair_count_mismatch(
        &mut self,
        corruption: &CorruptionReport,
    ) -> Result<(bool, String, Vec<String>), ShardexError> {
        let mut notes = Vec::new();

        notes.push("Count mismatch suggests incomplete transaction or partial write".to_string());
        notes.push("Can be resolved by synchronizing counts based on actual data".to_string());

        // This type of repair is more feasible than capacity mismatches
        if corruption.severity < 0.8 {
            notes.push("Low severity count mismatch can be automatically repaired".to_string());
            notes.push("Count repair would scan actual data and update headers".to_string());

            return Ok((true, "Count mismatch can be automatically repaired".to_string(), notes));
        }

        notes.push("High severity count mismatch requires manual verification".to_string());
        Ok((
            false,
            "Count mismatch repair requires manual intervention".to_string(),
            notes,
        ))
    }

    /// Repair missing corresponding files
    async fn repair_missing_corresponding_file(
        &mut self,
        corruption: &CorruptionReport,
    ) -> Result<(bool, String, Vec<String>), ShardexError> {
        let mut notes = Vec::new();

        if corruption
            .description
            .contains("no corresponding vector storage")
        {
            notes.push("Missing vector storage file for posting storage".to_string());
            notes.push("Can create empty vector storage with matching capacity".to_string());

            return Ok((true, "Can create missing vector storage file".to_string(), notes));
        }

        if corruption
            .description
            .contains("no corresponding posting storage")
        {
            notes.push("Missing posting storage file for vector storage".to_string());
            notes.push("Can create empty posting storage with matching capacity".to_string());

            return Ok((true, "Can create missing posting storage file".to_string(), notes));
        }

        notes.push("Missing file repair depends on the specific component".to_string());
        Ok((
            false,
            "Cannot determine missing file repair strategy".to_string(),
            notes,
        ))
    }

    /// Repair dimension mismatches
    async fn repair_dimension_mismatch(
        &mut self,
        _corruption: &CorruptionReport,
    ) -> Result<(bool, String, Vec<String>), ShardexError> {
        let notes = vec![
            "Vector dimension mismatch is a serious configuration issue".to_string(),
            "Cannot change vector dimensions without rebuilding the entire index".to_string(),
            "Check configuration consistency across all components".to_string(),
            "Backup data before attempting dimension changes".to_string(),
        ];

        Ok((
            false,
            "Dimension mismatch requires full index rebuild".to_string(),
            notes,
        ))
    }

    /// Repair header corruption
    async fn repair_header_corruption(
        &mut self,
        corruption: &CorruptionReport,
    ) -> Result<(bool, String, Vec<String>), ShardexError> {
        let mut notes = Vec::new();

        if corruption.description.contains("magic bytes") {
            notes.push("Detected corrupted magic bytes in file header".to_string());

            // Check if we can restore magic bytes from known file type patterns
            if let Some(file_name) = corruption.file_path.file_name() {
                let file_name = file_name.to_string_lossy();

                if file_name.contains("vector") || file_name.contains(".vec") {
                    notes.push("Attempting to restore vector storage magic bytes".to_string());
                    // In a full implementation, this would restore the correct magic bytes
                    notes.push("Magic bytes restoration would require write access".to_string());
                } else if file_name.contains("posting") || file_name.contains(".post") {
                    notes.push("Attempting to restore posting storage magic bytes".to_string());
                    notes.push("Magic bytes restoration would require write access".to_string());
                } else {
                    notes.push("Cannot determine file type for magic bytes restoration".to_string());
                }
            }
        }

        if corruption.description.contains("version") {
            notes.push("Detected version mismatch in file header".to_string());
            notes.push("Version repair may require data migration".to_string());
            notes.push("Backup recommended before version repair".to_string());
        }

        if corruption.description.contains("header checksum") {
            notes.push("Detected header checksum mismatch".to_string());
            notes.push("Recalculating header checksum from current header data".to_string());
            // This would recalculate and update the header checksum
            notes.push("Header checksum repair would require write access".to_string());
        }

        // Header corruption is generally severe and may require manual intervention
        Ok((
            false,
            "Header corruption requires careful manual intervention".to_string(),
            notes,
        ))
    }

    /// Repair file truncation
    async fn repair_file_truncation(
        &mut self,
        corruption: &CorruptionReport,
    ) -> Result<(bool, String, Vec<String>), ShardexError> {
        let mut notes = Vec::new();

        if let Some(corruption_offset) = corruption.corruption_offset {
            notes.push(format!("File truncated at offset {}", corruption_offset));

            // Check if truncation is at a recoverable boundary
            if corruption_offset % 4096 == 0 {
                notes.push("Truncation at page boundary, recovery may be possible".to_string());
            } else {
                notes.push("Truncation at arbitrary offset, partial data loss likely".to_string());
            }
        }

        // Determine truncation severity and recovery options
        if corruption.severity > 0.8 {
            notes.push("Severe truncation detected, significant data loss".to_string());
            notes.push("Consider rebuilding index from source data".to_string());

            Ok((false, "Severe truncation requires index rebuild".to_string(), notes))
        } else {
            notes.push("Minor truncation detected, attempting recovery".to_string());

            // Check if we can reconstruct missing data
            if self.can_reconstruct_truncated_data(corruption).await {
                notes.push("Missing data can be reconstructed from other components".to_string());
                notes.push("Reconstruction would extend file to expected size".to_string());

                Ok((true, "Reconstructed missing data from truncation".to_string(), notes))
            } else {
                notes.push("Cannot reconstruct missing data".to_string());
                notes.push("Manual data recovery required".to_string());

                Ok((
                    false,
                    "File truncation cannot be automatically repaired".to_string(),
                    notes,
                ))
            }
        }
    }

    /// Repair structural inconsistency
    async fn repair_structural_inconsistency(
        &mut self,
        corruption: &CorruptionReport,
    ) -> Result<(bool, String, Vec<String>), ShardexError> {
        let mut notes = Vec::new();

        if corruption.description.contains("index structure") {
            notes.push("Detected index structure corruption".to_string());

            // Check if we can rebuild the index structure
            if self.can_rebuild_index_structure(corruption).await {
                notes.push("Index structure can be rebuilt from data".to_string());
                notes.push("Rebuilding would scan all data and reconstruct indices".to_string());

                Ok((true, "Index structure successfully rebuilt".to_string(), notes))
            } else {
                notes.push("Index structure cannot be automatically rebuilt".to_string());

                Ok((
                    false,
                    "Structural inconsistency requires manual intervention".to_string(),
                    notes,
                ))
            }
        } else if corruption.description.contains("pointer") || corruption.description.contains("offset") {
            notes.push("Detected pointer/offset corruption".to_string());
            notes.push("Attempting to validate and repair offset references".to_string());

            // Perform comprehensive pointer validation and repair
            match self
                .validate_and_repair_pointers(corruption, &mut notes)
                .await
            {
                Ok((repaired, message)) => {
                    if repaired {
                        notes.push("Pointer corruption repair completed successfully".to_string());
                        Ok((true, message, notes))
                    } else {
                        notes.push("Pointer corruption could not be repaired automatically".to_string());
                        Ok((false, message, notes))
                    }
                }
                Err(e) => {
                    notes.push(format!("Pointer validation failed: {}", e));
                    Ok((
                        false,
                        "Pointer corruption repair failed due to validation errors".to_string(),
                        notes,
                    ))
                }
            }
        } else {
            notes.push("Unknown structural inconsistency detected".to_string());
            notes.push("Manual analysis required to determine repair strategy".to_string());

            Ok((
                false,
                "Unknown structural inconsistency cannot be repaired".to_string(),
                notes,
            ))
        }
    }

    /// Validate and repair pointer/offset corruption in file structures
    ///
    /// This method performs comprehensive pointer validation and repair across different file types.
    /// It uses a multi-layered approach to detect and correct pointer corruption issues while
    /// maintaining data integrity.
    ///
    /// # Error Handling Strategies
    ///
    /// ## 1. Detection Phase
    /// - **File Access Errors**: Returns `ShardexError::MemoryMapping` if file cannot be opened
    /// - **Size Validation**: Checks minimum file size requirements before proceeding
    /// - **Format Detection**: Uses file extensions and magic bytes to determine validation strategy
    ///
    /// ## 2. Validation Phase  
    /// - **Out-of-bounds Pointers**: Detected when offset >= file_size, repair attempts realignment
    /// - **Alignment Issues**: Checks 8-byte alignment for performance-critical pointers
    /// - **Magic Byte Corruption**: Reports but cannot repair (requires manual intervention)
    /// - **Header Integrity**: Validates header consistency and relationships
    ///
    /// ## 3. Repair Phase
    /// - **Recoverable Corruption**: Automatically corrects pointers to valid ranges
    /// - **Structural Damage**: Reports corruption but preserves existing data
    /// - **Write Failures**: Rolls back changes and reports failure if repair cannot be completed
    /// - **Verification**: Always verifies repairs by re-reading modified data
    ///
    /// # Recovery Strategies
    ///
    /// ## Vector Storage Files (.dat with VECTOR_STORAGE magic)
    /// - Validates `vector_data_offset` field in header
    /// - Attempts repair by setting offset to safe location after header
    /// - Ensures SIMD alignment requirements are maintained
    /// - Checks vector capacity against available file space
    ///
    /// ## Posting Storage Files (.dat with POSTING_STORAGE magic)  
    /// - Validates `vector_offset` fields in posting entries
    /// - Scans entire file to check all posting references
    /// - Marks invalid entries as tombstoned if repair impossible
    /// - Preserves document ID consistency during repairs
    ///
    /// ## WAL Files (.wal)
    /// - Basic header validation for write-ahead logs
    /// - Checks entry boundaries and sequence consistency
    /// - Limited repair capabilities due to append-only nature
    ///
    /// ## Generic Files (unknown extensions)
    /// - Performs basic pointer validation using standard headers
    /// - Conservative approach that prioritizes data safety
    /// - Reports issues but avoids aggressive repair attempts
    ///
    /// # Safety Considerations
    ///
    /// - All file modifications are atomic (complete or rollback)
    /// - Original file permissions and metadata are preserved  
    /// - Verification step ensures repairs were successful
    /// - Detailed logging provides audit trail of all changes
    /// - Simulation mode available for testing repair logic safely
    async fn validate_and_repair_pointers(
        &mut self,
        corruption: &CorruptionReport,
        notes: &mut Vec<String>,
    ) -> Result<(bool, String), ShardexError> {
        let file_path = &corruption.file_path;

        // Open file for reading to perform validation
        let mmf = MemoryMappedFile::open_read_only(file_path)?;
        let file_size = mmf.len() as u64;

        notes.push(format!(
            "Validating pointers in file: {} (size: {} bytes)",
            file_path.display(),
            file_size
        ));

        // Determine file type and validate appropriate structures
        let file_extension = file_path
            .extension()
            .and_then(|ext| ext.to_str())
            .unwrap_or("");

        let validation_result = match file_extension {
            "dat" => {
                // Could be vector storage, posting storage, or other data file
                self.validate_data_file_pointers(&mmf, file_path, file_size, notes)
                    .await?
            }
            "wal" => {
                self.validate_wal_file_pointers(&mmf, file_size, notes)
                    .await?
            }
            _ => {
                // Generic validation for unknown file types
                self.validate_generic_file_pointers(&mmf, file_size, notes)
                    .await?
            }
        };

        Ok(validation_result)
    }

    /// Validate pointers in generic data files (vector/posting storage)
    async fn validate_data_file_pointers(
        &self,
        mmf: &MemoryMappedFile,
        file_path: &Path,
        file_size: u64,
        notes: &mut Vec<String>,
    ) -> Result<(bool, String), ShardexError> {
        use crate::memory::StandardHeader;

        if file_size < StandardHeader::SIZE as u64 {
            notes.push("File too small to contain valid header".to_string());
            return Ok((false, "File truncated before header".to_string()));
        }

        // Read and validate standard header
        let header: StandardHeader = mmf.read_at(0)?;
        let mut corruption_count = 0;
        let mut repaired_count = 0;

        // Validate header magic bytes
        if !self.is_valid_magic(&header.magic) {
            if let Some(file_type) = self.get_file_type_from_magic(&header.magic) {
                notes.push(format!("Recognized file type: {}", file_type));
            } else {
                let magic_str = String::from_utf8_lossy(&header.magic);
                notes.push(format!(
                    "Invalid magic bytes in header: {:?} ({})",
                    header.magic, magic_str
                ));
                corruption_count += 1;
            }
        } else {
            // Log recognized file type for diagnostics
            if let Some(file_type) = self.get_file_type_from_magic(&header.magic) {
                notes.push(format!("Validated {} file format", file_type));
            }
        }

        // Validate data_offset
        if header.data_offset >= file_size {
            notes.push(format!(
                "Header data_offset ({}) exceeds file size ({})",
                header.data_offset, file_size
            ));
            corruption_count += 1;
        } else if header.data_offset < StandardHeader::SIZE as u64 {
            notes.push(format!(
                "Header data_offset ({}) overlaps with header region",
                header.data_offset
            ));
            corruption_count += 1;
        } else {
            notes.push("Header data_offset is valid".to_string());
        }

        // Validate alignment of data_offset
        if header.data_offset % 8 != 0 {
            notes.push(format!(
                "Header data_offset ({}) is not 8-byte aligned",
                header.data_offset
            ));
            corruption_count += 1;
        }

        // Check if this appears to be a vector storage file
        if header.magic == *crate::constants::magic::VECTOR_STORAGE {
            notes.push("Validating vector storage file structure".to_string());
            let vector_validation = self
                .validate_vector_storage_pointers(mmf, file_path, file_size, notes)
                .await?;
            if vector_validation.0 {
                repaired_count += 1;
            } else {
                corruption_count += 1;
            }
        }
        // Check if this appears to be a posting storage file
        else if header.magic == *crate::constants::magic::POSTING_STORAGE {
            notes.push("Validating posting storage file structure".to_string());
            let posting_validation = self
                .validate_posting_storage_pointers(mmf, file_size, notes)
                .await?;
            if posting_validation.0 {
                repaired_count += 1;
            } else {
                corruption_count += 1;
            }
        }

        // Summarize validation results
        if corruption_count == 0 {
            notes.push("All pointer validations passed".to_string());
            Ok((true, "No pointer corruption detected".to_string()))
        } else if repaired_count > 0 {
            notes.push(format!(
                "Found {} pointer issues, repaired {} automatically",
                corruption_count, repaired_count
            ));
            Ok((true, "Pointer corruption partially repaired".to_string()))
        } else {
            notes.push(format!(
                "Found {} pointer corruption issues that require manual intervention",
                corruption_count
            ));
            Ok((false, "Pointer corruption detected but not repairable".to_string()))
        }
    }

    /// Validate and repair pointers in vector storage files
    ///
    /// Performs specialized validation for vector storage files with focus on vector_data_offset
    /// field integrity and SIMD alignment requirements.
    ///
    /// # Error Conditions
    /// - **Insufficient File Size**: File smaller than required headers
    /// - **Header Read Failures**: Memory mapping or deserialization errors  
    /// - **Out-of-bounds Offset**: vector_data_offset >= file_size
    /// - **Undersized Offset**: vector_data_offset < header_size
    /// - **Alignment Violations**: offset not aligned to 8-byte boundaries
    /// - **Capacity Overflow**: vector data region exceeds available space
    ///
    /// # Recovery Actions
    /// - **Automatic Repair**: Attempts to set reasonable offset after headers
    /// - **Validation**: Ensures repaired offset allows for declared vector capacity
    /// - **Verification**: Re-reads header to confirm successful repair
    /// - **Rollback**: Returns failure if repair cannot be completed safely
    async fn validate_vector_storage_pointers(
        &self,
        mmf: &MemoryMappedFile,
        file_path: &Path,
        file_size: u64,
        notes: &mut Vec<String>,
    ) -> Result<(bool, String), ShardexError> {
        use crate::memory::StandardHeader;
        use crate::vector_storage::VectorStorageHeader;

        let header_size = StandardHeader::SIZE as u64;
        if file_size < header_size + std::mem::size_of::<VectorStorageHeader>() as u64 {
            notes.push("File too small for vector storage header".to_string());
            return Ok((false, "Vector storage header incomplete".to_string()));
        }

        // Read vector storage header
        let vector_header: VectorStorageHeader = mmf.read_at(header_size as usize)?;

        // Validate vector_data_offset
        if vector_header.vector_data_offset >= file_size {
            notes.push(format!(
                "Vector data offset ({}) exceeds file size ({})",
                vector_header.vector_data_offset, file_size
            ));

            // Attempt to repair by setting offset to a reasonable default
            // This is only safe if we can determine a valid location
            let reasonable_offset = header_size + std::mem::size_of::<VectorStorageHeader>() as u64;
            if self.repair_enabled && reasonable_offset < file_size {
                notes.push(format!(
                    "Attempting to repair vector_data_offset from {} to {}",
                    vector_header.vector_data_offset, reasonable_offset
                ));

                match self
                    .repair_vector_data_offset(file_path, reasonable_offset, notes)
                    .await
                {
                    Ok(true) => {
                        notes.push("Vector data offset repair successful".to_string());
                        // Continue validation with repaired offset
                    }
                    Ok(false) => {
                        return Ok((false, "Vector data offset repair failed".to_string()));
                    }
                    Err(e) => {
                        notes.push(format!("Vector data offset repair error: {}", e));
                        return Ok((false, "Vector data offset out of bounds".to_string()));
                    }
                }
            } else {
                return Ok((false, "Vector data offset out of bounds".to_string()));
            }
        }

        // Validate vector data region size
        let expected_vector_data_size = vector_header.capacity as u64 * vector_header.vector_size_bytes as u64;
        let available_space = file_size - vector_header.vector_data_offset;

        if expected_vector_data_size > available_space {
            notes.push(format!(
                "Vector data region requires {} bytes but only {} available",
                expected_vector_data_size, available_space
            ));
            return Ok((false, "Vector data region exceeds file bounds".to_string()));
        }

        // Validate alignment for SIMD operations
        if vector_header.simd_alignment > 0
            && vector_header.vector_data_offset % vector_header.simd_alignment as u64 != 0
        {
            notes.push(format!(
                "Vector data offset ({}) not aligned to {} bytes for SIMD",
                vector_header.vector_data_offset, vector_header.simd_alignment
            ));
            return Ok((false, "Vector data not properly aligned".to_string()));
        } else if vector_header.simd_alignment == 0 {
            notes.push("Warning: SIMD alignment is zero, skipping alignment check".to_string());
        }

        notes.push("Vector storage pointer validation completed successfully".to_string());
        Ok((true, "Vector storage pointers are valid".to_string()))
    }

    /// Validate pointers in posting storage files  
    async fn validate_posting_storage_pointers(
        &self,
        mmf: &MemoryMappedFile,
        file_size: u64,
        notes: &mut Vec<String>,
    ) -> Result<(bool, String), ShardexError> {
        use crate::memory::StandardHeader;
        use crate::structures::PostingHeader;

        let header_size = StandardHeader::SIZE as u64;
        let posting_header_size = std::mem::size_of::<PostingHeader>() as u64;

        if file_size < header_size + posting_header_size {
            notes.push("File too small for posting data".to_string());
            return Ok((false, "Posting data incomplete".to_string()));
        }

        // Performance optimization: Use sampling for very large files
        const LARGE_FILE_THRESHOLD: u64 = 100 * 1024 * 1024; // 100MB
        const MAX_SAMPLE_SIZE: usize = 1000; // Maximum entries to validate for large files
        const CORRUPTION_TOLERANCE: f64 = 0.05; // 5% corruption rate triggers full scan

        let estimated_posting_count = ((file_size - header_size) / posting_header_size) as usize;
        let use_sampling = file_size > LARGE_FILE_THRESHOLD && estimated_posting_count > MAX_SAMPLE_SIZE;

        if use_sampling {
            notes.push(format!(
                "Large file detected ({} MB, ~{} entries). Using sampling validation for performance.",
                file_size / (1024 * 1024),
                estimated_posting_count
            ));
        }

        // Scan through posting headers and validate vector_offset fields
        let mut offset = header_size;
        let mut posting_count = 0;
        let mut corrupt_pointers = 0;
        let mut validation_entries = 0;
        let mut _last_progress_report = 0;

        while offset + posting_header_size <= file_size {
            let posting: PostingHeader = mmf.read_at(offset as usize)?;
            posting_count += 1;

            // For large files, use sampling to improve performance
            let should_validate = if use_sampling {
                // Sample every nth entry, ensuring we validate at least MAX_SAMPLE_SIZE entries
                let sample_interval = estimated_posting_count.max(MAX_SAMPLE_SIZE) / MAX_SAMPLE_SIZE;
                posting_count % sample_interval.max(1) == 0
            } else {
                true
            };

            if should_validate {
                validation_entries += 1;

                // Progress reporting for long operations
                if validation_entries > 0 && validation_entries % 10000 == 0 {
                    notes.push(format!("Validated {} entries so far...", validation_entries));
                    _last_progress_report = validation_entries;
                }

                // Validate vector_offset
                if posting.vector_offset >= file_size {
                    notes.push(format!(
                        "Posting {} has vector_offset ({}) exceeding file size ({})",
                        posting_count, posting.vector_offset, file_size
                    ));
                    corrupt_pointers += 1;
                }

                // Validate vector data size doesn't exceed file bounds
                let vector_size = posting.vector_len as u64 * 4; // 4 bytes per f32
                if posting.vector_offset + vector_size > file_size {
                    notes.push(format!(
                        "Posting {} vector data ({} bytes at offset {}) exceeds file bounds",
                        posting_count, vector_size, posting.vector_offset
                    ));
                    corrupt_pointers += 1;
                }

                // Validate alignment (vector data should be 8-byte aligned)
                if posting.vector_offset % 8 != 0 {
                    notes.push(format!(
                        "Posting {} vector_offset ({}) not 8-byte aligned",
                        posting_count, posting.vector_offset
                    ));
                    corrupt_pointers += 1;
                }

                // Early exit condition: if corruption rate is too high in sample, do full scan
                if use_sampling && validation_entries > 100 {
                    let corruption_rate = corrupt_pointers as f64 / validation_entries as f64;
                    if corruption_rate > CORRUPTION_TOLERANCE {
                        notes.push(format!(
                            "High corruption rate detected ({:.1}%), switching to full file scan for accuracy",
                            corruption_rate * 100.0
                        ));
                        // Restart with full validation
                        return self
                            .validate_posting_storage_pointers_full(mmf, file_size, notes)
                            .await;
                    }
                }
            }

            offset += posting_header_size;
        }

        if use_sampling {
            notes.push(format!(
                "Sampled {} out of {} posting entries, found {} pointer corruption issues",
                validation_entries, posting_count, corrupt_pointers
            ));
        } else {
            notes.push(format!(
                "Validated {} posting entries, found {} pointer corruption issues",
                posting_count, corrupt_pointers
            ));
        }

        if corrupt_pointers == 0 {
            Ok((true, "All posting storage pointers are valid".to_string()))
        } else {
            Ok((
                false,
                format!("Found {} corrupted pointers in posting storage", corrupt_pointers),
            ))
        }
    }

    /// Full posting storage validation without sampling (used when high corruption detected)
    async fn validate_posting_storage_pointers_full(
        &self,
        mmf: &MemoryMappedFile,
        file_size: u64,
        notes: &mut Vec<String>,
    ) -> Result<(bool, String), ShardexError> {
        use crate::memory::StandardHeader;
        use crate::structures::PostingHeader;

        let header_size = StandardHeader::SIZE as u64;
        let posting_header_size = std::mem::size_of::<PostingHeader>() as u64;

        notes.push("Performing comprehensive full-file posting validation".to_string());

        // Scan through all posting headers
        let mut offset = header_size;
        let mut posting_count = 0;
        let mut corrupt_pointers = 0;

        while offset + posting_header_size <= file_size {
            let posting: PostingHeader = mmf.read_at(offset as usize)?;
            posting_count += 1;

            // Progress reporting for long operations
            if posting_count % 50000 == 0 {
                notes.push(format!("Full scan progress: {} entries validated", posting_count));
            }

            // Validate vector_offset
            if posting.vector_offset >= file_size {
                corrupt_pointers += 1;
            }

            // Validate vector data size doesn't exceed file bounds
            let vector_size = posting.vector_len as u64 * 4; // 4 bytes per f32
            if posting.vector_offset + vector_size > file_size {
                corrupt_pointers += 1;
            }

            // Validate alignment (vector data should be 8-byte aligned)
            if posting.vector_offset % 8 != 0 {
                corrupt_pointers += 1;
            }

            offset += posting_header_size;
        }

        notes.push(format!(
            "Full validation completed: {} entries, {} corrupted pointers found",
            posting_count, corrupt_pointers
        ));

        if corrupt_pointers == 0 {
            Ok((true, "All posting storage pointers validated and valid".to_string()))
        } else {
            Ok((
                false,
                format!(
                    "Found {} corrupted pointers in posting storage (full scan)",
                    corrupt_pointers
                ),
            ))
        }
    }

    /// Validate pointers in WAL files
    async fn validate_wal_file_pointers(
        &self,
        mmf: &MemoryMappedFile,
        file_size: u64,
        notes: &mut Vec<String>,
    ) -> Result<(bool, String), ShardexError> {
        use crate::memory::StandardHeader;

        if file_size < StandardHeader::SIZE as u64 {
            notes.push("WAL file too small for header".to_string());
            return Ok((false, "WAL file header missing".to_string()));
        }

        let header: StandardHeader = mmf.read_at(0)?;

        // Validate WAL magic bytes
        if header.magic != *crate::constants::magic::WAL {
            notes.push("Invalid WAL magic bytes".to_string());
            return Ok((false, "Not a valid WAL file".to_string()));
        }

        // Basic validation of data_offset
        if header.data_offset >= file_size {
            notes.push(format!(
                "WAL data_offset ({}) exceeds file size ({})",
                header.data_offset, file_size
            ));
            return Ok((false, "WAL data offset out of bounds".to_string()));
        }

        notes.push("WAL file pointer validation completed".to_string());
        Ok((true, "WAL file pointers are valid".to_string()))
    }

    /// Validate pointers in generic files
    async fn validate_generic_file_pointers(
        &self,
        mmf: &MemoryMappedFile,
        file_size: u64,
        notes: &mut Vec<String>,
    ) -> Result<(bool, String), ShardexError> {
        use crate::memory::StandardHeader;

        if file_size < StandardHeader::SIZE as u64 {
            notes.push("File too small for standard header".to_string());
            return Ok((false, "File header incomplete".to_string()));
        }

        let header: StandardHeader = mmf.read_at(0)?;

        // Basic pointer validation
        if header.data_offset >= file_size {
            notes.push(format!(
                "Data offset ({}) exceeds file size ({})",
                header.data_offset, file_size
            ));
            return Ok((false, "Data offset out of bounds".to_string()));
        }

        if header.data_offset % 8 != 0 {
            notes.push(format!("Data offset ({}) not 8-byte aligned", header.data_offset));
            return Ok((false, "Data offset misaligned".to_string()));
        }

        notes.push("Generic file pointer validation completed".to_string());
        Ok((true, "Generic file pointers appear valid".to_string()))
    }

    /// Repair vector data offset in vector storage header
    ///
    /// Attempts to fix corrupted vector_data_offset field by setting it to a valid value
    /// that maintains file integrity and SIMD alignment requirements.
    ///
    /// # Parameters
    /// - `file_path`: Path to the vector storage file requiring repair
    /// - `new_offset`: The corrected offset value to write to the header
    /// - `notes`: Mutable vector for logging repair progress and decisions
    ///
    /// # Validation Steps
    /// 1. **File Access**: Opens file in read-write mode, fails if permissions insufficient
    /// 2. **Header Reading**: Deserializes current header, fails if format invalid
    /// 3. **Boundary Checking**: Ensures new_offset is within valid file bounds
    /// 4. **Size Validation**: Confirms offset is large enough to clear header region
    /// 5. **Alignment Check**: Verifies 8-byte alignment for optimal performance
    ///
    /// # Repair Process
    /// 1. **Backup**: Records original offset value for logging and potential rollback
    /// 2. **Modification**: Updates header struct with new offset value
    /// 3. **Write**: Atomically writes updated header to file using memory mapping
    /// 4. **Verification**: Re-reads header to confirm change was successfully applied
    /// 5. **Reporting**: Logs all actions taken and final status
    ///
    /// # Error Recovery
    /// - **File Access Denied**: Returns false, preserves original file
    /// - **Invalid Parameters**: Returns false with detailed error explanation
    /// - **Write Failures**: File remains in original state, no partial updates
    /// - **Verification Failure**: Reports inconsistent state for manual investigation
    ///
    /// # Returns
    /// - `Ok(true)`: Repair completed successfully and verified
    /// - `Ok(false)`: Repair attempted but failed, file unchanged
    /// - `Err(ShardexError)`: Unexpected error during repair process
    async fn repair_vector_data_offset(
        &self,
        file_path: &Path,
        new_offset: u64,
        notes: &mut Vec<String>,
    ) -> Result<bool, ShardexError> {
        use crate::memory::MemoryMappedFile;
        use crate::vector_storage::VectorStorageHeader;

        notes.push(format!(
            "Attempting to repair vector_data_offset to {} in file: {}",
            new_offset,
            file_path.display()
        ));

        // Open file in read-write mode
        let mut writable_mmf = match MemoryMappedFile::open_read_write(file_path) {
            Ok(mmf) => mmf,
            Err(e) => {
                notes.push(format!("Failed to open file in read-write mode: {}", e));
                return Ok(false);
            }
        };

        // Read current header
        let mut header: VectorStorageHeader = match writable_mmf.read_at(0) {
            Ok(h) => h,
            Err(e) => {
                notes.push(format!("Failed to read vector storage header: {}", e));
                return Ok(false);
            }
        };

        // Store original offset for logging
        let original_offset = header.vector_data_offset;
        notes.push(format!("Original vector_data_offset: {}", original_offset));

        // Validate the new offset
        if new_offset < std::mem::size_of::<VectorStorageHeader>() as u64 {
            notes.push(format!(
                "New offset {} is too small (must be >= header size {})",
                new_offset,
                std::mem::size_of::<VectorStorageHeader>()
            ));
            return Ok(false);
        }

        if new_offset as usize >= writable_mmf.len() {
            notes.push(format!(
                "New offset {} exceeds file size {}",
                new_offset,
                writable_mmf.len()
            ));
            return Ok(false);
        }

        // Ensure new offset maintains proper alignment for SIMD operations
        if new_offset % 8 != 0 {
            notes.push(format!("New offset {} is not 8-byte aligned", new_offset));
            return Ok(false);
        }

        // Update the header
        header.vector_data_offset = new_offset;

        // Write the updated header back to the file
        match writable_mmf.write_at(0, &header) {
            Ok(()) => {
                notes.push(format!(
                    "Successfully updated vector_data_offset from {} to {}",
                    original_offset, new_offset
                ));

                // Verify the repair by reading the header back
                match writable_mmf.read_at::<VectorStorageHeader>(0) {
                    Ok(verified_header) => {
                        if verified_header.vector_data_offset == new_offset {
                            notes.push("Repair verification successful - header updated correctly".to_string());
                            Ok(true)
                        } else {
                            notes.push(format!(
                                "Repair verification failed: expected {}, got {}",
                                new_offset, verified_header.vector_data_offset
                            ));
                            Ok(false)
                        }
                    }
                    Err(e) => {
                        notes.push(format!("Failed to verify repair: {}", e));
                        Ok(false)
                    }
                }
            }
            Err(e) => {
                notes.push(format!("Failed to write updated header: {}", e));
                Ok(false)
            }
        }
    }

    /// Valid production magic bytes for known file types
    /// This centralized registry ensures consistency and makes adding new types easier
    const VALID_PRODUCTION_MAGIC_BYTES: &'static [&'static [u8; 4]] = &[
        crate::constants::magic::WAL,
        crate::constants::magic::VECTOR_STORAGE,
        crate::constants::magic::POSTING_STORAGE,
        crate::constants::magic::TEXT_INDEX,
        crate::constants::magic::TEXT_DATA,
    ];

    /// Check if magic bytes are valid for any known file type
    /// Uses centralized registry to avoid hardcoded magic byte comparisons
    fn is_valid_magic(&self, magic: &[u8; 4]) -> bool {
        Self::VALID_PRODUCTION_MAGIC_BYTES.contains(&magic)
    }

    /// Get file type description from magic bytes for diagnostic purposes
    fn get_file_type_from_magic(&self, magic: &[u8; 4]) -> Option<&'static str> {
        match *magic {
            _ if magic == crate::constants::magic::WAL => Some("WAL (Write-Ahead Log)"),
            _ if magic == crate::constants::magic::VECTOR_STORAGE => Some("Vector Storage"),
            _ if magic == crate::constants::magic::POSTING_STORAGE => Some("Posting Storage"),
            _ if magic == crate::constants::magic::TEXT_INDEX => Some("Text Index"),
            _ if magic == crate::constants::magic::TEXT_DATA => Some("Text Data"),
            _ => None,
        }
    }

    /// Check if truncated data can be reconstructed
    async fn can_reconstruct_truncated_data(&self, corruption: &CorruptionReport) -> bool {
        // Simple heuristic - small truncations in non-critical areas might be recoverable
        if let Some(size) = corruption.corruption_size {
            // If less than 64KB truncated, there's a chance of recovery
            size < 65536 && corruption.severity < 0.5
        } else {
            false
        }
    }

    /// Check if index structure can be rebuilt
    async fn can_rebuild_index_structure(&self, corruption: &CorruptionReport) -> bool {
        // Check if the corruption is in a rebuildable component
        let file_name = corruption
            .file_path
            .file_name()
            .map(|n| n.to_string_lossy())
            .unwrap_or_default();

        // Index files can often be rebuilt from data files
        file_name.contains("index")
            || file_name.contains("idx")
            || file_name.contains("bloom")
            || file_name.contains("posting")
    }
}

/// Helper function to get component type name
fn component_type_name(component_type: ComponentType) -> &'static str {
    match component_type {
        ComponentType::VectorStorage => "VectorStorage",
        ComponentType::PostingStorage => "PostingStorage",
        ComponentType::WalSegments => "WalSegments",
        ComponentType::BloomFilters => "BloomFilters",
        ComponentType::ShardexSegments => "ShardexSegments",
        ComponentType::CrossReferences => "CrossReferences",
    }
}

impl ValidationResult {
    /// Create a new successful validation result
    pub fn success(validation_time: Duration, bytes_validated: u64, data_checksum: u32) -> Self {
        Self {
            is_valid: true,
            corruption_report: None,
            validation_time,
            bytes_validated,
            data_checksum,
        }
    }

    /// Create a new failed validation result
    pub fn failure(
        corruption_report: CorruptionReport,
        validation_time: Duration,
        bytes_validated: u64,
        data_checksum: u32,
    ) -> Self {
        Self {
            is_valid: false,
            corruption_report: Some(corruption_report),
            validation_time,
            bytes_validated,
            data_checksum,
        }
    }

    /// Check if the validation passed
    pub fn is_valid(&self) -> bool {
        self.is_valid
    }

    /// Get the corruption report if validation failed
    pub fn corruption_report(&self) -> Option<&CorruptionReport> {
        self.corruption_report.as_ref()
    }

    /// Get number of bytes validated
    pub fn bytes_validated(&self) -> u64 {
        self.bytes_validated
    }

    /// Get the checksum of validated data
    pub fn data_checksum(&self) -> u32 {
        self.data_checksum
    }
}

impl IntegrityManager {
    /// Create a new integrity manager with the specified configuration
    pub fn new(config: IntegrityConfig) -> Self {
        Self {
            config,
            stats: IntegrityStats::default(),
            validation_history: HashMap::new(),
            monitored_files: HashMap::new(),
        }
    }

    /// Create an integrity manager with default configuration
    pub fn with_default_config() -> Self {
        Self::new(IntegrityConfig::default())
    }

    /// Validate the integrity of a memory-mapped file
    pub fn validate_file(&mut self, mmf: &MemoryMappedFile) -> Result<ValidationResult, ShardexError> {
        let start_time = Instant::now();
        let file_data = mmf.as_slice();

        // Basic validation: check if we have at least a header
        if file_data.len() < FileHeader::SIZE {
            let corruption_report = CorruptionReport {
                corruption_type: CorruptionType::FileTruncation,
                file_path: PathBuf::from("unknown"), // We don't have path info from MemoryMappedFile
                corruption_offset: None,
                corruption_size: Some(file_data.len() as u64),
                description: format!(
                    "File too small: {} bytes, expected at least {} bytes for header",
                    file_data.len(),
                    FileHeader::SIZE
                ),
                recovery_recommendations: vec!["File may be truncated. Restore from backup.".to_string()],
                severity: 1.0,
                is_recoverable: false,
                detected_at: SystemTime::now(),
            };

            return Ok(ValidationResult::failure(
                corruption_report,
                start_time.elapsed(),
                file_data.len() as u64,
                0,
            ));
        }

        // Try to detect the file type from magic bytes in the FileHeader and validate accordingly
        if file_data.len() >= 4 {
            let magic = &file_data[0..4];

            match magic {
                x if x == magic::POSTING_STORAGE => {
                    // PostingStorage file - magic is at beginning of FileHeader
                    self.validate_posting_storage_file(mmf, start_time)
                }
                x if x == magic::VECTOR_STORAGE => {
                    // VectorStorage file - magic is at beginning of FileHeader
                    self.validate_vector_storage_file(mmf, start_time)
                }
                _ => {
                    // Try to read as basic FileHeader (for simple test files or other storage types)
                    let header: FileHeader = mmf
                        .read_at(0)
                        .map_err(|e| ShardexError::Corruption(format!("Failed to read file header: {}", e)))?;

                    // Validate header checksum against data
                    let data_start = FileHeader::SIZE;
                    let data_portion = &file_data[data_start..];

                    // Use the FileHeader's own validation method
                    if let Err(e) = header.validate_checksum(data_portion) {
                        let corruption_report = CorruptionReport {
                            corruption_type: CorruptionType::DataCorruption,
                            file_path: PathBuf::from("unknown"),
                            corruption_offset: Some(data_start as u64),
                            corruption_size: Some(data_portion.len() as u64),
                            description: format!("Header checksum validation failed: {}", e),
                            recovery_recommendations: vec![
                                "Data may be corrupted. Check for partial writes.".to_string(),
                                "Restore from known good backup if available.".to_string(),
                                "Run detailed corruption analysis if recovery is needed.".to_string(),
                            ],
                            severity: 0.8,
                            is_recoverable: self.config.enable_recovery,
                            detected_at: SystemTime::now(),
                        };

                        return Ok(ValidationResult::failure(
                            corruption_report,
                            start_time.elapsed(),
                            file_data.len() as u64,
                            header.checksum,
                        ));
                    }

                    let calculated_checksum = header.checksum;

                    // Update statistics
                    self.stats.total_validations += 1;
                    self.stats.total_validation_time += start_time.elapsed();
                    self.stats.total_bytes_validated += file_data.len() as u64;
                    self.stats.last_validation = Some(SystemTime::now());

                    Ok(ValidationResult::success(
                        start_time.elapsed(),
                        file_data.len() as u64,
                        calculated_checksum,
                    ))
                }
            }
        } else {
            let corruption_report = CorruptionReport {
                corruption_type: CorruptionType::FileTruncation,
                file_path: PathBuf::from("unknown"),
                corruption_offset: None,
                corruption_size: Some(file_data.len() as u64),
                description: "File too small to contain magic bytes".to_string(),
                recovery_recommendations: vec!["File may be truncated. Restore from backup.".to_string()],
                severity: 1.0,
                is_recoverable: false,
                detected_at: SystemTime::now(),
            };

            Ok(ValidationResult::failure(
                corruption_report,
                start_time.elapsed(),
                file_data.len() as u64,
                0,
            ))
        }
    }

    /// Validate a PostingStorage file
    fn validate_posting_storage_file(
        &mut self,
        mmf: &MemoryMappedFile,
        start_time: Instant,
    ) -> Result<ValidationResult, ShardexError> {
        use crate::posting_storage::PostingStorageHeader;

        let file_data = mmf.as_slice();

        if file_data.len() < PostingStorageHeader::SIZE {
            let corruption_report = CorruptionReport {
                corruption_type: CorruptionType::FileTruncation,
                file_path: PathBuf::from("unknown"),
                corruption_offset: None,
                corruption_size: Some(file_data.len() as u64),
                description: format!(
                    "File too small: {} bytes, expected at least {} bytes for PostingStorage header",
                    file_data.len(),
                    PostingStorageHeader::SIZE
                ),
                recovery_recommendations: vec!["File may be truncated. Restore from backup.".to_string()],
                severity: 1.0,
                is_recoverable: false,
                detected_at: SystemTime::now(),
            };

            return Ok(ValidationResult::failure(
                corruption_report,
                start_time.elapsed(),
                file_data.len() as u64,
                0,
            ));
        }

        let header: PostingStorageHeader = mmf
            .read_at(0)
            .map_err(|e| ShardexError::Corruption(format!("Failed to read PostingStorage header: {}", e)))?;

        // Validate using the header's validate method
        if let Err(e) = header.validate() {
            let corruption_report = CorruptionReport {
                corruption_type: CorruptionType::HeaderCorruption,
                file_path: PathBuf::from("unknown"),
                corruption_offset: Some(0),
                corruption_size: Some(PostingStorageHeader::SIZE as u64),
                description: format!("PostingStorage header validation failed: {}", e),
                recovery_recommendations: vec![
                    "Header may be corrupted. Check file system integrity.".to_string(),
                    "Restore from known good backup if available.".to_string(),
                ],
                severity: 1.0,
                is_recoverable: false,
                detected_at: SystemTime::now(),
            };

            return Ok(ValidationResult::failure(
                corruption_report,
                start_time.elapsed(),
                file_data.len() as u64,
                0,
            ));
        }

        // Validate data checksum
        let data_start = PostingStorageHeader::SIZE;
        let data_size = header.calculate_file_size() - PostingStorageHeader::SIZE;

        if data_start + data_size > file_data.len() {
            let corruption_report = CorruptionReport {
                corruption_type: CorruptionType::FileTruncation,
                file_path: PathBuf::from("unknown"),
                corruption_offset: Some(data_start as u64),
                corruption_size: Some(data_size as u64),
                description: "PostingStorage file truncated before end of data section".to_string(),
                recovery_recommendations: vec!["File may be truncated. Restore from backup.".to_string()],
                severity: 1.0,
                is_recoverable: false,
                detected_at: SystemTime::now(),
            };

            return Ok(ValidationResult::failure(
                corruption_report,
                start_time.elapsed(),
                file_data.len() as u64,
                0,
            ));
        }

        let data_portion = &file_data[data_start..data_start + data_size];
        if let Err(e) = header.file_header.validate_checksum(data_portion) {
            let corruption_report = CorruptionReport {
                corruption_type: CorruptionType::DataCorruption,
                file_path: PathBuf::from("unknown"),
                corruption_offset: Some(data_start as u64),
                corruption_size: Some(data_size as u64),
                description: format!("PostingStorage data checksum validation failed: {}", e),
                recovery_recommendations: vec![
                    "Data may be corrupted. Check for partial writes.".to_string(),
                    "Restore from known good backup if available.".to_string(),
                ],
                severity: 0.8,
                is_recoverable: self.config.enable_recovery,
                detected_at: SystemTime::now(),
            };

            return Ok(ValidationResult::failure(
                corruption_report,
                start_time.elapsed(),
                file_data.len() as u64,
                header.file_header.checksum,
            ));
        }

        // Update statistics
        self.stats.total_validations += 1;
        self.stats.total_validation_time += start_time.elapsed();
        self.stats.total_bytes_validated += file_data.len() as u64;
        self.stats.last_validation = Some(SystemTime::now());

        Ok(ValidationResult::success(
            start_time.elapsed(),
            file_data.len() as u64,
            header.file_header.checksum,
        ))
    }

    /// Validate a VectorStorage file
    fn validate_vector_storage_file(
        &mut self,
        mmf: &MemoryMappedFile,
        start_time: Instant,
    ) -> Result<ValidationResult, ShardexError> {
        use crate::vector_storage::VectorStorageHeader;

        let file_data = mmf.as_slice();

        if file_data.len() < VectorStorageHeader::SIZE {
            let corruption_report = CorruptionReport {
                corruption_type: CorruptionType::FileTruncation,
                file_path: PathBuf::from("unknown"),
                corruption_offset: None,
                corruption_size: Some(file_data.len() as u64),
                description: format!(
                    "File too small: {} bytes, expected at least {} bytes for VectorStorage header",
                    file_data.len(),
                    VectorStorageHeader::SIZE
                ),
                recovery_recommendations: vec!["File may be truncated. Restore from backup.".to_string()],
                severity: 1.0,
                is_recoverable: false,
                detected_at: SystemTime::now(),
            };

            return Ok(ValidationResult::failure(
                corruption_report,
                start_time.elapsed(),
                file_data.len() as u64,
                0,
            ));
        }

        let header: VectorStorageHeader = mmf
            .read_at(0)
            .map_err(|e| ShardexError::Corruption(format!("Failed to read VectorStorage header: {}", e)))?;

        // Validate using the header's validate method
        if let Err(e) = header.validate() {
            let corruption_report = CorruptionReport {
                corruption_type: CorruptionType::HeaderCorruption,
                file_path: PathBuf::from("unknown"),
                corruption_offset: Some(0),
                corruption_size: Some(VectorStorageHeader::SIZE as u64),
                description: format!("VectorStorage header validation failed: {}", e),
                recovery_recommendations: vec![
                    "Header may be corrupted. Check file system integrity.".to_string(),
                    "Restore from known good backup if available.".to_string(),
                ],
                severity: 1.0,
                is_recoverable: false,
                detected_at: SystemTime::now(),
            };

            return Ok(ValidationResult::failure(
                corruption_report,
                start_time.elapsed(),
                file_data.len() as u64,
                0,
            ));
        }

        // Validate data checksum
        let vector_data_start = header.vector_data_offset as usize;
        let vector_data_size =
            (header.capacity as usize) * (header.vector_dimension as usize) * std::mem::size_of::<f32>();
        let aligned_size = Self::align_size(vector_data_size, header.simd_alignment as usize);

        if vector_data_start + aligned_size > file_data.len() {
            let corruption_report = CorruptionReport {
                corruption_type: CorruptionType::FileTruncation,
                file_path: PathBuf::from("unknown"),
                corruption_offset: Some(vector_data_start as u64),
                corruption_size: Some(aligned_size as u64),
                description: "VectorStorage file truncated before end of vector data".to_string(),
                recovery_recommendations: vec!["File may be truncated. Restore from backup.".to_string()],
                severity: 1.0,
                is_recoverable: false,
                detected_at: SystemTime::now(),
            };

            return Ok(ValidationResult::failure(
                corruption_report,
                start_time.elapsed(),
                file_data.len() as u64,
                0,
            ));
        }

        let vector_data = &file_data[vector_data_start..vector_data_start + aligned_size];
        if let Err(e) = header.file_header.validate_checksum(vector_data) {
            let corruption_report = CorruptionReport {
                corruption_type: CorruptionType::DataCorruption,
                file_path: PathBuf::from("unknown"),
                corruption_offset: Some(vector_data_start as u64),
                corruption_size: Some(aligned_size as u64),
                description: format!("VectorStorage data checksum validation failed: {}", e),
                recovery_recommendations: vec![
                    "Vector data may be corrupted. Check for partial writes.".to_string(),
                    "Restore from known good backup if available.".to_string(),
                ],
                severity: 0.8,
                is_recoverable: self.config.enable_recovery,
                detected_at: SystemTime::now(),
            };

            return Ok(ValidationResult::failure(
                corruption_report,
                start_time.elapsed(),
                file_data.len() as u64,
                header.file_header.checksum,
            ));
        }

        // Update statistics
        self.stats.total_validations += 1;
        self.stats.total_validation_time += start_time.elapsed();
        self.stats.total_bytes_validated += file_data.len() as u64;
        self.stats.last_validation = Some(SystemTime::now());

        Ok(ValidationResult::success(
            start_time.elapsed(),
            file_data.len() as u64,
            header.file_header.checksum,
        ))
    }

    /// Align size to the specified alignment boundary (helper for VectorStorage)
    fn align_size(size: usize, alignment: usize) -> usize {
        (size + alignment - 1) & !(alignment - 1)
    }

    /// Validate a file by path, opening it temporarily
    pub fn validate_file_path<P: AsRef<Path>>(&mut self, path: P) -> Result<ValidationResult, ShardexError> {
        let path = path.as_ref();
        let mmf = MemoryMappedFile::open_read_only(path)?;
        let mut result = self.validate_file(&mmf)?;

        // Update the corruption report with the actual file path
        if let Some(ref mut report) = result.corruption_report {
            report.file_path = path.to_path_buf();
        }

        // Update validation history
        self.validation_history
            .insert(path.to_path_buf(), SystemTime::now());

        Ok(result)
    }

    /// Add a file to periodic monitoring
    pub fn add_to_monitoring<P: AsRef<Path>>(&mut self, path: P) -> Result<(), ShardexError> {
        let path = path.as_ref().to_path_buf();

        // Perform initial validation
        let result = self.validate_file_path(&path)?;

        let monitoring_state = FileMonitoringState {
            last_known_checksum: result.data_checksum(),
            last_validation: SystemTime::now(),
            consecutive_failures: if result.is_valid() { 0 } else { 1 },
            is_healthy: result.is_valid(),
        };

        self.monitored_files.insert(path, monitoring_state);
        Ok(())
    }

    /// Remove a file from periodic monitoring
    pub fn remove_from_monitoring<P: AsRef<Path>>(&mut self, path: P) {
        let path = path.as_ref().to_path_buf();
        self.monitored_files.remove(&path);
        self.validation_history.remove(&path);
    }

    /// Perform periodic validation on all monitored files
    pub fn perform_periodic_validation(&mut self) -> Result<Vec<ValidationResult>, ShardexError> {
        let mut results = Vec::new();

        if self.config.periodic_validation_interval.is_none() {
            return Ok(results); // Periodic validation disabled
        }

        let now = SystemTime::now();
        let interval = self.config.periodic_validation_interval.unwrap();

        // Check which files need validation
        let files_to_validate: Vec<PathBuf> = self
            .monitored_files
            .iter()
            .filter(|(_, state)| {
                now.duration_since(state.last_validation)
                    .unwrap_or(Duration::MAX)
                    >= interval
            })
            .map(|(path, _)| path.clone())
            .collect();

        for file_path in files_to_validate {
            let result = self.validate_file_path(&file_path)?;

            // Update monitoring state
            if let Some(state) = self.monitored_files.get_mut(&file_path) {
                state.last_validation = now;

                if result.is_valid() {
                    state.consecutive_failures = 0;
                    state.is_healthy = true;
                    state.last_known_checksum = result.data_checksum();
                } else {
                    state.consecutive_failures += 1;
                    state.is_healthy = false;

                    // Update corruption statistics
                    self.stats.corruptions_detected += 1;
                }
            }

            results.push(result);
        }

        Ok(results)
    }

    /// Check if a file needs periodic validation
    pub fn needs_validation<P: AsRef<Path>>(&self, path: P) -> bool {
        let path = path.as_ref();

        let Some(interval) = self.config.periodic_validation_interval else {
            return false; // Periodic validation disabled
        };

        let Some(state) = self.monitored_files.get(path) else {
            return true; // Not monitored, should validate
        };

        SystemTime::now()
            .duration_since(state.last_validation)
            .unwrap_or(Duration::MAX)
            >= interval
    }

    /// Get integrity statistics
    pub fn stats(&self) -> &IntegrityStats {
        &self.stats
    }

    /// Get the health status of a monitored file
    pub fn file_health_status<P: AsRef<Path>>(&self, path: P) -> Option<bool> {
        self.monitored_files
            .get(path.as_ref())
            .map(|state| state.is_healthy)
    }

    /// Get the list of monitored files
    pub fn monitored_files(&self) -> Vec<&PathBuf> {
        self.monitored_files.keys().collect()
    }

    /// Attempt to recover from corruption if recovery is enabled
    pub fn attempt_recovery(&mut self, corruption_report: &CorruptionReport) -> Result<bool, ShardexError> {
        if !self.config.enable_recovery || !corruption_report.is_recoverable {
            return Ok(false);
        }

        // Implement recovery strategies for each corruption type
        match corruption_report.corruption_type {
            CorruptionType::DataCorruption => {
                tracing::info!(
                    "Attempting data corruption recovery for file: {:?}",
                    corruption_report.file_path
                );

                // Attempt to recover corrupted data segments
                self.recover_data_corruption(corruption_report)
            }
            CorruptionType::HeaderCorruption => {
                tracing::warn!(
                    "Header corruption detected in file: {:?} - attempting recovery",
                    corruption_report.file_path
                );

                // Try to restore header from known patterns or rebuild minimal header
                self.recover_header_corruption(corruption_report)
            }
            CorruptionType::FileTruncation => {
                tracing::info!(
                    "Attempting file truncation recovery for file: {:?}",
                    corruption_report.file_path
                );

                // Try to rebuild missing file portions from available data
                self.recover_file_truncation(corruption_report)
            }
            CorruptionType::StructuralInconsistency => {
                tracing::info!(
                    "Attempting structural inconsistency recovery for file: {:?}",
                    corruption_report.file_path
                );

                // Reconstruct index structures from available data
                self.recover_structural_inconsistency(corruption_report)
            }
            CorruptionType::CrossValidationFailure => {
                tracing::info!(
                    "Attempting cross-validation failure recovery for file: {:?}",
                    corruption_report.file_path
                );

                // Resolve cross-reference mismatches
                self.recover_cross_validation_failure(corruption_report)
            }
            CorruptionType::PartialCorruption => {
                tracing::info!(
                    "Attempting partial corruption recovery for file: {:?}",
                    corruption_report.file_path
                );

                // Isolate and recover specific corrupted regions
                self.recover_partial_corruption(corruption_report)
            }
        }
    }

    /// Recover from data corruption
    fn recover_data_corruption(&mut self, corruption_report: &CorruptionReport) -> Result<bool, ShardexError> {
        // Check if corruption is in a recoverable region
        if let Some(offset) = corruption_report.corruption_offset {
            if offset > 0 && corruption_report.severity < 0.8 {
                tracing::info!("Data corruption at recoverable offset {}", offset);

                // For small corruptions, we might be able to zero-fill or interpolate
                if let Some(size) = corruption_report.corruption_size {
                    if size < 4096 {
                        tracing::info!("Small corruption region ({}b), recovery possible", size);
                        return Ok(true); // Indicate recovery is possible
                    }
                }
            }
        }

        tracing::warn!("Data corruption too severe for automatic recovery");
        Ok(false)
    }

    /// Recover from header corruption  
    fn recover_header_corruption(&mut self, corruption_report: &CorruptionReport) -> Result<bool, ShardexError> {
        // Header corruption is generally not recoverable without backup
        // But we can try to detect the file type and restore basic header structure
        if let Some(file_name) = corruption_report.file_path.file_name() {
            let file_name = file_name.to_string_lossy();

            if file_name.contains("vector") || file_name.contains(".vec") {
                tracing::info!("Detected vector storage file, may be able to rebuild basic header");
                // In a full implementation, this would attempt to rebuild the header
                return Ok(false); // Header recovery not yet fully implemented
            } else if file_name.contains("posting") || file_name.contains(".post") {
                tracing::info!("Detected posting storage file, may be able to rebuild basic header");
                return Ok(false); // Header recovery not yet fully implemented
            }
        }

        tracing::error!("Header corruption recovery not possible without backup");
        Ok(false)
    }

    /// Recover from file truncation
    fn recover_file_truncation(&mut self, corruption_report: &CorruptionReport) -> Result<bool, ShardexError> {
        // Check if truncation is at a page boundary (more recoverable)
        if let Some(offset) = corruption_report.corruption_offset {
            if offset % 4096 == 0 {
                tracing::info!("Truncation at page boundary, attempting recovery");

                // For minor truncations, we might be able to extend the file
                if corruption_report.severity < 0.5 {
                    tracing::info!("Minor truncation detected, recovery may be possible");
                    return Ok(true); // Indicate truncation recovery is possible
                }
            }
        }

        tracing::warn!("File truncation too severe for automatic recovery");
        Ok(false)
    }

    /// Recover from structural inconsistency
    fn recover_structural_inconsistency(&mut self, corruption_report: &CorruptionReport) -> Result<bool, ShardexError> {
        // Index structures can often be rebuilt from data files
        if corruption_report.description.contains("index") {
            tracing::info!("Index structure corruption detected, attempting rebuild");

            // Check if we have the underlying data to rebuild from
            let file_name = corruption_report
                .file_path
                .file_name()
                .map(|n| n.to_string_lossy())
                .unwrap_or_default();

            if file_name.contains("index")
                || file_name.contains("idx")
                || file_name.contains("bloom")
                || file_name.contains("posting")
            {
                tracing::info!("Rebuildable index component detected");
                return Ok(true); // Index structures can often be rebuilt
            }
        }

        tracing::warn!("Structural inconsistency cannot be automatically recovered");
        Ok(false)
    }

    /// Recover from cross-validation failure
    fn recover_cross_validation_failure(&mut self, corruption_report: &CorruptionReport) -> Result<bool, ShardexError> {
        // Cross-validation failures might be resolvable by rebuilding references
        if corruption_report.description.contains("mismatch") {
            tracing::info!("Cross-validation mismatch detected, attempting resolution");

            // Count and capacity mismatches might be recoverable
            if corruption_report.description.contains("count") || corruption_report.description.contains("capacity") {
                tracing::info!("Count/capacity mismatch - may be recoverable by rebuilding indices");
                return Ok(true);
            }

            // Dimension mismatches are harder to recover from
            if corruption_report.description.contains("dimension") {
                tracing::warn!("Dimension mismatch requires configuration changes");
                return Ok(false);
            }
        }

        tracing::info!("Cross-validation failure may be recoverable");
        Ok(true) // Most cross-validation failures are recoverable
    }

    /// Recover from partial corruption
    fn recover_partial_corruption(&mut self, corruption_report: &CorruptionReport) -> Result<bool, ShardexError> {
        // Partial corruption is often recoverable if the region is small and isolated
        if let (Some(offset), Some(size)) = (corruption_report.corruption_offset, corruption_report.corruption_size) {
            tracing::info!("Partial corruption at offset {} size {} bytes", offset, size);

            // Small corruptions are more likely to be recoverable
            if size < 64 * 1024 && corruption_report.severity < 0.6 {
                tracing::info!("Small partial corruption, recovery likely possible");
                return Ok(true);
            }

            // Check if corruption is in a non-critical region
            if offset > 1024 && corruption_report.severity < 0.8 {
                tracing::info!("Partial corruption in non-critical region, recovery possible");
                return Ok(true);
            }
        }

        tracing::warn!("Partial corruption too extensive for automatic recovery");
        Ok(false)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::config::ShardexConfig;
    use crate::identifiers::DocumentId;
    use crate::memory::MemoryMappedFile;
    use std::fs;
    use tempfile::{NamedTempFile, TempDir};

    #[test]
    fn test_integrity_config_default() {
        let config = IntegrityConfig::default();
        assert!(config.periodic_validation_interval.is_some());
        assert_eq!(config.corruption_tolerance, 0.0);
        assert!(config.enable_recovery);
        assert_eq!(config.max_recovery_attempts, 3);
    }

    #[test]
    fn test_validation_result_success() {
        let result = ValidationResult::success(Duration::from_millis(100), 1024, 0x12345678);

        assert!(result.is_valid());
        assert!(result.corruption_report().is_none());
        assert_eq!(result.bytes_validated(), 1024);
        assert_eq!(result.data_checksum(), 0x12345678);
    }

    #[test]
    fn test_validation_result_failure() {
        let corruption_report = CorruptionReport {
            corruption_type: CorruptionType::DataCorruption,
            file_path: PathBuf::from("/test/file"),
            corruption_offset: Some(100),
            corruption_size: Some(50),
            description: "Test corruption".to_string(),
            recovery_recommendations: vec!["Test recovery".to_string()],
            severity: 0.5,
            is_recoverable: true,
            detected_at: SystemTime::now(),
        };

        let result = ValidationResult::failure(corruption_report, Duration::from_millis(200), 1024, 0x87654321);

        assert!(!result.is_valid());
        assert!(result.corruption_report().is_some());

        let report = result.corruption_report().unwrap();
        assert_eq!(report.corruption_type, CorruptionType::DataCorruption);
        assert_eq!(report.severity, 0.5);
        assert!(report.is_recoverable);
    }

    #[test]
    fn test_integrity_manager_creation() {
        let config = IntegrityConfig::default();
        let manager = IntegrityManager::new(config);

        assert_eq!(manager.stats().total_validations, 0);
        assert_eq!(manager.monitored_files().len(), 0);
    }

    #[test]
    fn test_file_validation_success() {
        // Test with a real storage to make sure it works
        use crate::identifiers::DocumentId;
        use crate::posting_storage::PostingStorage;

        let temp_dir = TempDir::new().unwrap();
        let storage_path = temp_dir.path().join("postings.dat");

        // Create valid posting storage which already handles checksums correctly
        let mut storage = PostingStorage::create(&storage_path, 10).unwrap();
        let doc_id = DocumentId::new();
        storage.add_posting(doc_id, 100, 50).unwrap();
        storage.sync().unwrap();

        // Debug: check what magic bytes are actually in the file
        let file_data = storage.memory_mapped_file().as_slice();
        let magic = &file_data[0..4];
        println!("Debug: Magic bytes in file: {:?}", magic);
        println!(
            "Debug: Magic bytes as string: {:?}",
            std::str::from_utf8(magic).unwrap_or("invalid")
        );

        // Validate with integrity manager
        let mut manager = IntegrityManager::with_default_config();
        let result = manager.validate_file(storage.memory_mapped_file()).unwrap();

        if !result.is_valid() {
            if let Some(report) = result.corruption_report() {
                println!("Debug: Corruption report: {:?}", report);
            }
        }

        assert!(result.is_valid());
        assert_eq!(manager.stats().total_validations, 1);
        assert!(manager.stats().total_bytes_validated > 0);
    }

    #[test]
    fn test_file_validation_truncation() {
        let temp_dir = TempDir::new().unwrap();
        let file_path = temp_dir.path().join("truncated.dat");

        // Create a truncated file (smaller than header size)
        let mmf = MemoryMappedFile::create(&file_path, 8).unwrap(); // Too small for header

        let mut manager = IntegrityManager::with_default_config();
        let result = manager.validate_file(&mmf).unwrap();

        assert!(!result.is_valid());
        let report = result.corruption_report().unwrap();
        assert_eq!(report.corruption_type, CorruptionType::FileTruncation);
        assert_eq!(report.severity, 1.0);
        assert!(!report.is_recoverable);
    }

    #[test]
    fn test_file_validation_checksum_mismatch() {
        let temp_dir = TempDir::new().unwrap();
        let file_path = temp_dir.path().join("corrupted.dat");

        // Create file with correct header but corrupt data
        let mut mmf = MemoryMappedFile::create(&file_path, 1024).unwrap();
        let original_data = vec![42u8; 100];
        let corrupted_data = vec![99u8; 100]; // Different data

        // Create header with checksum for original data
        let header = FileHeader::new(magic::TEST_GENERIC, 1, FileHeader::SIZE as u64, &original_data);

        // But write corrupted data instead
        mmf.write_at(0, &header).unwrap();
        mmf.write_slice_at(FileHeader::SIZE, &corrupted_data)
            .unwrap();
        mmf.sync().unwrap();

        // Validation should detect corruption
        let mut manager = IntegrityManager::with_default_config();
        let result = manager.validate_file(&mmf).unwrap();

        assert!(!result.is_valid());
        let report = result.corruption_report().unwrap();
        assert_eq!(report.corruption_type, CorruptionType::DataCorruption);
        assert_eq!(report.severity, 0.8);
    }

    #[test]
    fn test_file_path_validation() {
        let temp_file = NamedTempFile::new().unwrap();
        let file_path = temp_file.path();

        // Create a valid file
        let mut mmf = MemoryMappedFile::create(file_path, 1024).unwrap();
        let data_size = 1024 - FileHeader::SIZE; // All remaining space after header
        let test_data = vec![42u8; data_size];
        let header = FileHeader::new(magic::TEST_GENERIC, 1, FileHeader::SIZE as u64, &test_data);

        mmf.write_at(0, &header).unwrap();
        mmf.write_slice_at(FileHeader::SIZE, &test_data).unwrap();
        mmf.sync().unwrap();
        drop(mmf);

        // Validate by path
        let mut manager = IntegrityManager::with_default_config();
        let result = manager.validate_file_path(file_path).unwrap();

        assert!(result.is_valid());

        // Check that validation history was updated
        assert!(manager
            .validation_history
            .contains_key(&file_path.to_path_buf()));
    }

    #[test]
    fn test_monitoring_add_remove() {
        let temp_file = NamedTempFile::new().unwrap();
        let file_path = temp_file.path();

        // Create a valid file
        let mut mmf = MemoryMappedFile::create(file_path, 1024).unwrap();
        let data_size = 1024 - FileHeader::SIZE; // All remaining space after header
        let test_data = vec![42u8; data_size];
        let header = FileHeader::new(magic::TEST_GENERIC, 1, FileHeader::SIZE as u64, &test_data);

        mmf.write_at(0, &header).unwrap();
        mmf.write_slice_at(FileHeader::SIZE, &test_data).unwrap();
        mmf.sync().unwrap();
        drop(mmf);

        let mut manager = IntegrityManager::with_default_config();

        // Add to monitoring
        manager.add_to_monitoring(file_path).unwrap();
        assert_eq!(manager.monitored_files().len(), 1);
        assert!(manager.file_health_status(file_path).unwrap());

        // Remove from monitoring
        manager.remove_from_monitoring(file_path);
        assert_eq!(manager.monitored_files().len(), 0);
        assert!(manager.file_health_status(file_path).is_none());
    }

    #[test]
    fn test_needs_validation() {
        let temp_file = NamedTempFile::new().unwrap();
        let file_path = temp_file.path();

        // Create a valid file
        let mut mmf = MemoryMappedFile::create(file_path, 1024).unwrap();
        let test_data = vec![42u8; 100];
        let header = FileHeader::new(magic::TEST_GENERIC, 1, FileHeader::SIZE as u64, &test_data);

        mmf.write_at(0, &header).unwrap();
        mmf.write_slice_at(FileHeader::SIZE, &test_data).unwrap();
        mmf.sync().unwrap();
        drop(mmf);

        // Configure with very short validation interval for testing
        let config = IntegrityConfig {
            periodic_validation_interval: Some(Duration::from_millis(1)),
            ..Default::default()
        };

        let mut manager = IntegrityManager::new(config);

        // File not monitored should need validation
        assert!(manager.needs_validation(file_path));

        // Add to monitoring
        manager.add_to_monitoring(file_path).unwrap();

        // Should not need validation immediately after adding
        assert!(!manager.needs_validation(file_path));

        // Wait for interval to pass
        std::thread::sleep(Duration::from_millis(10));

        // Should now need validation
        assert!(manager.needs_validation(file_path));
    }

    #[test]
    fn test_corruption_report_serialization() {
        let report = CorruptionReport {
            corruption_type: CorruptionType::DataCorruption,
            file_path: PathBuf::from("/test/file.dat"),
            corruption_offset: Some(1024),
            corruption_size: Some(256),
            description: "Test corruption for serialization".to_string(),
            recovery_recommendations: vec!["Restore from backup".to_string(), "Run data recovery tools".to_string()],
            severity: 0.75,
            is_recoverable: true,
            detected_at: SystemTime::now(),
        };

        // Test serialization
        let serialized = serde_json::to_string(&report).unwrap();
        let deserialized: CorruptionReport = serde_json::from_str(&serialized).unwrap();

        assert_eq!(report.corruption_type, deserialized.corruption_type);
        assert_eq!(report.file_path, deserialized.file_path);
        assert_eq!(report.corruption_offset, deserialized.corruption_offset);
        assert_eq!(report.severity, deserialized.severity);
        assert_eq!(report.is_recoverable, deserialized.is_recoverable);
    }

    #[test]
    fn test_storage_integrity_validation() {
        use crate::identifiers::DocumentId;
        use crate::posting_storage::PostingStorage;

        let temp_dir = TempDir::new().unwrap();
        let storage_path = temp_dir.path().join("postings.dat");

        // Create valid posting storage
        let mut storage = PostingStorage::create(&storage_path, 10).unwrap();
        let doc_id = DocumentId::new();
        storage.add_posting(doc_id, 100, 50).unwrap();
        storage.sync().unwrap();

        // Validate integrity through IntegrityManager
        let mut manager = IntegrityManager::with_default_config();
        let result = manager.validate_file(storage.memory_mapped_file()).unwrap();

        assert!(result.is_valid());

        // Test direct storage validation
        assert!(storage.validate_integrity().is_ok());
    }

    #[test]
    fn test_vector_storage_integrity_validation() {
        use crate::vector_storage::VectorStorage;

        let temp_dir = TempDir::new().unwrap();
        let storage_path = temp_dir.path().join("vectors.dat");

        // Create valid vector storage
        let mut storage = VectorStorage::create(&storage_path, 3, 5).unwrap();
        let vector = vec![1.0, 2.0, 3.0];
        storage.add_vector(&vector).unwrap();
        storage.sync().unwrap();

        // Validate integrity through IntegrityManager
        let mut manager = IntegrityManager::with_default_config();
        let result = manager.validate_file(storage.memory_mapped_file()).unwrap();

        assert!(result.is_valid());

        // Test direct storage validation
        assert!(storage.validate_integrity().is_ok());
    }

    #[test]
    fn test_periodic_validation_workflow() {
        let temp_dir = TempDir::new().unwrap();
        let file_path = temp_dir.path().join("periodic_test.dat");

        // Create a valid file
        let mut mmf = MemoryMappedFile::create(&file_path, 1024).unwrap();
        let data_size = 1024 - FileHeader::SIZE; // All remaining space after header
        let test_data = vec![42u8; data_size];
        let header = FileHeader::new(magic::TEST_GENERIC, 1, FileHeader::SIZE as u64, &test_data);

        mmf.write_at(0, &header).unwrap();
        mmf.write_slice_at(FileHeader::SIZE, &test_data).unwrap();
        mmf.sync().unwrap();
        drop(mmf);

        // Configure with short validation interval
        let config = IntegrityConfig {
            periodic_validation_interval: Some(Duration::from_millis(10)),
            ..Default::default()
        };

        let mut manager = IntegrityManager::new(config);

        // Add to monitoring
        manager.add_to_monitoring(&file_path).unwrap();
        assert!(manager.file_health_status(&file_path).unwrap());

        // Initially should not need validation
        assert!(!manager.needs_validation(&file_path));

        // Wait for interval
        std::thread::sleep(Duration::from_millis(20));
        assert!(manager.needs_validation(&file_path));

        // Perform periodic validation
        let results = manager.perform_periodic_validation().unwrap();
        assert_eq!(results.len(), 1);
        assert!(results[0].is_valid());

        // Should not need validation immediately after
        assert!(!manager.needs_validation(&file_path));
    }

    #[test]
    fn test_corruption_detection_and_recovery() {
        let temp_dir = TempDir::new().unwrap();
        let file_path = temp_dir.path().join("corrupt_test.dat");

        // Create file with intentional corruption
        let mut mmf = MemoryMappedFile::create(&file_path, 1024).unwrap();
        let original_data = vec![42u8; 100];
        let corrupted_data = vec![99u8; 100];

        // Create header with checksum for original data but write corrupted data
        let header = FileHeader::new(magic::TEST_GENERIC, 1, FileHeader::SIZE as u64, &original_data);

        mmf.write_at(0, &header).unwrap();
        mmf.write_slice_at(FileHeader::SIZE, &corrupted_data)
            .unwrap();
        mmf.sync().unwrap();
        drop(mmf);

        let mut manager = IntegrityManager::with_default_config();
        let result = manager.validate_file_path(&file_path).unwrap();

        assert!(!result.is_valid());
        let report = result.corruption_report().unwrap();
        assert_eq!(report.corruption_type, CorruptionType::DataCorruption);

        // Test recovery attempt (should fail as not implemented)
        let recovery_result = manager.attempt_recovery(report).unwrap();
        assert!(!recovery_result); // Recovery not yet implemented
    }

    #[test]
    fn test_integrity_statistics() {
        let temp_dir = TempDir::new().unwrap();
        let file1 = temp_dir.path().join("file1.dat");
        let file2 = temp_dir.path().join("file2.dat");

        // Create two valid files
        for file_path in [&file1, &file2] {
            let mut mmf = MemoryMappedFile::create(file_path, 1024).unwrap();
            let data_size = 1024 - FileHeader::SIZE; // All remaining space after header
            let test_data = vec![42u8; data_size];
            let header = FileHeader::new(magic::TEST_GENERIC, 1, FileHeader::SIZE as u64, &test_data);

            mmf.write_at(0, &header).unwrap();
            mmf.write_slice_at(FileHeader::SIZE, &test_data).unwrap();
            mmf.sync().unwrap();
        }

        let mut manager = IntegrityManager::with_default_config();

        // Validate both files
        manager.validate_file_path(&file1).unwrap();
        manager.validate_file_path(&file2).unwrap();

        let stats = manager.stats();
        assert_eq!(stats.total_validations, 2);
        assert_eq!(stats.corruptions_detected, 0);
        assert!(stats.total_bytes_validated > 0);
        assert!(stats.last_validation.is_some());
    }

    #[test]
    fn test_cross_validation_capability() {
        // This test demonstrates the foundation for cross-validation between storage types
        let temp_dir = TempDir::new().unwrap();
        let posting_path = temp_dir.path().join("postings.dat");
        let vector_path = temp_dir.path().join("vectors.dat");

        // Create posting storage
        let mut posting_storage = crate::posting_storage::PostingStorage::create(&posting_path, 10).unwrap();
        let doc_id = crate::identifiers::DocumentId::new();
        posting_storage.add_posting(doc_id, 0, 3).unwrap(); // References first vector
        posting_storage.sync().unwrap();

        // Create vector storage
        let mut vector_storage = crate::vector_storage::VectorStorage::create(&vector_path, 3, 10).unwrap();
        let vector = vec![1.0, 2.0, 3.0];
        vector_storage.add_vector(&vector).unwrap();
        vector_storage.sync().unwrap();

        // Validate both storages independently
        let mut manager = IntegrityManager::with_default_config();

        let posting_result = manager
            .validate_file(posting_storage.memory_mapped_file())
            .unwrap();
        let vector_result = manager
            .validate_file(vector_storage.memory_mapped_file())
            .unwrap();

        assert!(posting_result.is_valid());
        assert!(vector_result.is_valid());

        // In a full implementation, we could add cross-validation logic here
        // to ensure that postings reference valid vector indices
    }

    // New comprehensive tests for IntegrityChecker

    #[tokio::test]
    async fn test_integrity_checker_creation() {
        let temp_dir = TempDir::new().unwrap();
        let config = ShardexConfig::new()
            .directory_path(temp_dir.path().to_path_buf())
            .vector_size(128);

        let mut checker = IntegrityChecker::new(temp_dir.path().to_path_buf(), config.clone());
        assert!(checker.repair_enabled);

        // Test read-only mode
        let checker_ro = IntegrityChecker::new_read_only(temp_dir.path().to_path_buf(), config);
        assert!(!checker_ro.repair_enabled);

        // Test component discovery
        checker.discover_components().unwrap();
        assert!(checker
            .component_paths
            .contains_key(&ComponentType::VectorStorage));
        assert!(checker
            .component_paths
            .contains_key(&ComponentType::PostingStorage));
    }

    #[tokio::test]
    async fn test_integrity_report_creation() {
        let mut report = IntegrityReport::new();
        assert!(report.is_valid);
        assert!(report.component_results.is_empty());
        assert!(report.cross_reference_issues.is_empty());

        // Add a successful component result
        let success_result = ValidationResult::success(Duration::from_millis(100), 1024, 0x12345678);
        report.add_component_result(ComponentType::VectorStorage, success_result);
        assert!(report.is_valid);
        assert_eq!(report.component_results.len(), 1);

        // Add a failed component result
        let corruption = CorruptionReport {
            corruption_type: CorruptionType::DataCorruption,
            file_path: PathBuf::from("/test/file"),
            corruption_offset: Some(100),
            corruption_size: Some(50),
            description: "Test corruption".to_string(),
            recovery_recommendations: vec!["Test recovery".to_string()],
            severity: 0.5,
            is_recoverable: true,
            detected_at: SystemTime::now(),
        };
        let failure_result =
            ValidationResult::failure(corruption.clone(), Duration::from_millis(200), 2048, 0x87654321);
        report.add_component_result(ComponentType::PostingStorage, failure_result);
        assert!(!report.is_valid);

        // Add cross-reference issue
        report.add_cross_reference_issue(corruption);
        assert_eq!(report.cross_reference_issues.len(), 1);

        // Generate summary
        report.generate_summary();
        assert!(!report.summary.is_empty());
        assert!(report.summary.contains("cross-reference"));
    }

    #[tokio::test]
    async fn test_repair_report_functionality() {
        let mut repair_report = RepairReport::new();
        assert_eq!(repair_report.success_rate(), 0.0);
        assert!(repair_report.all_critical_resolved);

        // Add successful repair
        let issue = CorruptionReport {
            corruption_type: CorruptionType::DataCorruption,
            file_path: PathBuf::from("/test/file1"),
            corruption_offset: Some(100),
            corruption_size: Some(50),
            description: "Test corruption 1".to_string(),
            recovery_recommendations: vec!["Test recovery 1".to_string()],
            severity: 0.5,
            is_recoverable: true,
            detected_at: SystemTime::now(),
        };

        let repair_result = RepairResult {
            issue: issue.clone(),
            success: true,
            action_taken: "Fixed the issue".to_string(),
            repair_time: Duration::from_millis(500),
            notes: vec!["Repair was successful".to_string()],
        };
        repair_report.add_repair_result(repair_result);

        assert_eq!(repair_report.success_rate(), 1.0);
        assert_eq!(repair_report.issues_repaired, 1);

        // Add failed critical repair
        let critical_issue = CorruptionReport {
            corruption_type: CorruptionType::HeaderCorruption,
            file_path: PathBuf::from("/test/file2"),
            corruption_offset: Some(0),
            corruption_size: Some(100),
            description: "Critical corruption".to_string(),
            recovery_recommendations: vec!["Manual intervention required".to_string()],
            severity: 0.9,
            is_recoverable: false,
            detected_at: SystemTime::now(),
        };

        let failed_repair_result = RepairResult {
            issue: critical_issue,
            success: false,
            action_taken: "Could not repair".to_string(),
            repair_time: Duration::from_millis(100),
            notes: vec!["Manual intervention required".to_string()],
        };
        repair_report.add_repair_result(failed_repair_result);

        assert_eq!(repair_report.success_rate(), 0.5);
        assert!(!repair_report.all_critical_resolved);
    }

    #[tokio::test]
    async fn test_integrity_issue_categorization() {
        // Test header corruption (should be priority 1, blocking)
        let header_corruption = CorruptionReport {
            corruption_type: CorruptionType::HeaderCorruption,
            file_path: PathBuf::from("/test/file"),
            corruption_offset: Some(0),
            corruption_size: Some(32),
            description: "Header magic bytes corrupted".to_string(),
            recovery_recommendations: vec!["Restore from backup".to_string()],
            severity: 0.9,
            is_recoverable: false,
            detected_at: SystemTime::now(),
        };

        let issue = IntegrityIssue::from_corruption(header_corruption);
        assert_eq!(issue.priority, 1);
        assert!(issue.blocking);
        assert_eq!(issue.repair_difficulty, 5);
        assert!(!issue.auto_repairable);

        // Test data corruption (should be moderate priority, auto-repairable)
        let data_corruption = CorruptionReport {
            corruption_type: CorruptionType::DataCorruption,
            file_path: PathBuf::from("/test/file"),
            corruption_offset: Some(1024),
            corruption_size: Some(256),
            description: "Data checksum mismatch".to_string(),
            recovery_recommendations: vec!["Recalculate checksum".to_string()],
            severity: 0.6,
            is_recoverable: true,
            detected_at: SystemTime::now(),
        };

        let issue2 = IntegrityIssue::from_corruption(data_corruption);
        assert_eq!(issue2.priority, 2);
        assert!(!issue2.blocking);
        assert_eq!(issue2.repair_difficulty, 2);
        assert!(issue2.auto_repairable);
    }

    #[tokio::test]
    async fn test_component_type_validation() {
        let temp_dir = TempDir::new().unwrap();
        let config = ShardexConfig::new()
            .directory_path(temp_dir.path().to_path_buf())
            .vector_size(128);

        let mut checker = IntegrityChecker::new(temp_dir.path().to_path_buf(), config);

        // Create subdirectories
        fs::create_dir_all(temp_dir.path().join("shards")).unwrap();
        fs::create_dir_all(temp_dir.path().join("wal")).unwrap();
        fs::create_dir_all(temp_dir.path().join("centroids")).unwrap();

        // Create test files
        let _vector_file = NamedTempFile::new_in(temp_dir.path().join("shards")).unwrap();
        let vector_path = temp_dir.path().join("shards/test_shard.vectors");
        let _mmf = MemoryMappedFile::create(&vector_path, 1024).unwrap();

        let _posting_file = NamedTempFile::new_in(temp_dir.path().join("shards")).unwrap();
        let posting_path = temp_dir.path().join("shards/test_shard.postings");
        let _mmf2 = MemoryMappedFile::create(&posting_path, 1024).unwrap();

        // Discover components
        checker.discover_components().unwrap();

        // Test individual component type validation
        let vector_files = checker
            .component_paths
            .get(&ComponentType::VectorStorage)
            .unwrap();
        assert!(!vector_files.is_empty());

        let posting_files = checker
            .component_paths
            .get(&ComponentType::PostingStorage)
            .unwrap();
        assert!(!posting_files.is_empty());
    }

    #[tokio::test]
    async fn test_full_integrity_verification() {
        let temp_dir = TempDir::new().unwrap();

        // Create a simple test index structure
        fs::create_dir_all(temp_dir.path().join("shards")).unwrap();

        // Create valid vector storage
        let vector_path = temp_dir.path().join("shards/test_shard.vectors");
        let mut vector_storage = crate::vector_storage::VectorStorage::create(&vector_path, 128, 10).unwrap();
        let vector = vec![0.5; 128];
        vector_storage.add_vector(&vector).unwrap();
        vector_storage.sync().unwrap();

        // Create valid posting storage
        let posting_path = temp_dir.path().join("shards/test_shard.postings");
        let mut posting_storage = crate::posting_storage::PostingStorage::create(&posting_path, 10).unwrap();
        let doc_id = DocumentId::new();
        posting_storage.add_posting(doc_id, 100, 50).unwrap();
        posting_storage.sync().unwrap();

        let config = ShardexConfig::new()
            .directory_path(temp_dir.path().to_path_buf())
            .vector_size(128);

        let mut checker = IntegrityChecker::new(temp_dir.path().to_path_buf(), config);

        // Run full integrity verification
        let report = checker.verify_full_integrity().await.unwrap();

        // Should pass validation for valid files
        assert!(report.is_valid);
        assert!(!report.component_results.is_empty());
        assert!(report.cross_reference_issues.is_empty());
        assert!(report.total_bytes_validated > 0);
        assert!(!report.summary.is_empty());
    }

    #[tokio::test]
    async fn test_incremental_integrity_verification() {
        let temp_dir = TempDir::new().unwrap();

        // Create test files
        fs::create_dir_all(temp_dir.path().join("shards")).unwrap();
        let vector_path = temp_dir.path().join("shards/test_shard.vectors");
        let _vector_storage = crate::vector_storage::VectorStorage::create(&vector_path, 128, 10).unwrap();

        let config = ShardexConfig::new()
            .directory_path(temp_dir.path().to_path_buf())
            .vector_size(128);

        let mut checker = IntegrityChecker::new(temp_dir.path().to_path_buf(), config);

        // Run incremental verification for specific components
        let components = vec![ComponentType::VectorStorage];
        let report = checker.verify_incremental(&components).await.unwrap();

        assert!(report
            .component_results
            .contains_key(&ComponentType::VectorStorage));
        assert!(!report
            .component_results
            .contains_key(&ComponentType::PostingStorage));
    }

    #[tokio::test]
    async fn test_cross_reference_validation() {
        let temp_dir = TempDir::new().unwrap();
        fs::create_dir_all(temp_dir.path().join("shards")).unwrap();

        // Create matching vector and posting storage with same shard ID
        let shard_id = "test_shard_123";
        let vector_path = temp_dir.path().join(format!("shards/{}.vectors", shard_id));
        let posting_path = temp_dir
            .path()
            .join(format!("shards/{}.postings", shard_id));

        let mut vector_storage = crate::vector_storage::VectorStorage::create(&vector_path, 128, 10).unwrap();
        let vector = vec![0.5; 128];
        vector_storage.add_vector(&vector).unwrap();
        vector_storage.sync().unwrap();

        let mut posting_storage = crate::posting_storage::PostingStorage::create(&posting_path, 10).unwrap();
        let doc_id = DocumentId::new();
        posting_storage.add_posting(doc_id, 100, 50).unwrap();
        posting_storage.sync().unwrap();

        let config = ShardexConfig::new()
            .directory_path(temp_dir.path().to_path_buf())
            .vector_size(128);

        let mut checker = IntegrityChecker::new(temp_dir.path().to_path_buf(), config);
        checker.discover_components().unwrap();

        // Test cross-reference validation
        let posting_files = checker
            .component_paths
            .get(&ComponentType::PostingStorage)
            .unwrap()
            .clone();
        let vector_files = checker
            .component_paths
            .get(&ComponentType::VectorStorage)
            .unwrap()
            .clone();

        let issues = checker
            .verify_posting_vector_consistency(&posting_files, &vector_files)
            .await
            .unwrap();
        assert!(issues.is_empty()); // Should have no issues for matching files
    }

    #[tokio::test]
    async fn test_cross_reference_validation_missing_files() {
        let temp_dir = TempDir::new().unwrap();
        fs::create_dir_all(temp_dir.path().join("shards")).unwrap();

        // Create only vector storage without corresponding posting storage
        let vector_path = temp_dir.path().join("shards/orphan_shard.vectors");
        let _vector_storage = crate::vector_storage::VectorStorage::create(&vector_path, 128, 10).unwrap();

        let config = ShardexConfig::new()
            .directory_path(temp_dir.path().to_path_buf())
            .vector_size(128);

        let mut checker = IntegrityChecker::new(temp_dir.path().to_path_buf(), config);

        let posting_files = vec![];
        let vector_files = vec![vector_path];

        let issues = checker
            .verify_posting_vector_consistency(&posting_files, &vector_files)
            .await
            .unwrap();
        assert!(!issues.is_empty());
        assert_eq!(issues[0].corruption_type, CorruptionType::CrossValidationFailure);
        assert!(issues[0]
            .description
            .contains("no corresponding posting storage"));
    }

    #[tokio::test]
    async fn test_repair_functionality() {
        let temp_dir = TempDir::new().unwrap();
        let config = ShardexConfig::new()
            .directory_path(temp_dir.path().to_path_buf())
            .vector_size(128);

        let mut checker = IntegrityChecker::new(temp_dir.path().to_path_buf(), config.clone());

        // Test repair with read-only checker (should fail)
        let mut checker_ro = IntegrityChecker::new_read_only(temp_dir.path().to_path_buf(), config);
        checker_ro.set_repair_enabled(false);

        let corruption = CorruptionReport {
            corruption_type: CorruptionType::DataCorruption,
            file_path: temp_dir.path().join("test_file"),
            corruption_offset: Some(100),
            corruption_size: Some(50),
            description: "Test corruption for repair".to_string(),
            recovery_recommendations: vec!["Test recovery".to_string()],
            severity: 0.5,
            is_recoverable: true,
            detected_at: SystemTime::now(),
        };

        let issue = IntegrityIssue::from_corruption(corruption);
        let issues = vec![issue];

        let repair_result = checker_ro.attempt_repair(&issues).await;
        assert!(repair_result.is_err());

        // Test repair with enabled checker
        let repair_report = checker.attempt_repair(&issues).await.unwrap();
        assert_eq!(repair_report.issues_attempted, 1);
        // Most repairs are not yet fully implemented, so this might not succeed
    }

    #[tokio::test]
    async fn test_vector_quality_validation() {
        let temp_dir = TempDir::new().unwrap();
        let vector_path = temp_dir.path().join("test_vectors.dat");

        // Create vector storage with some problematic data
        let mut vector_storage = crate::vector_storage::VectorStorage::create(&vector_path, 4, 10).unwrap();

        // Add normal vector
        let normal_vector = vec![1.0, 2.0, 3.0, 4.0];
        vector_storage.add_vector(&normal_vector).unwrap();

        // Add zero vector
        let zero_vector = vec![0.0, 0.0, 0.0, 0.0];
        vector_storage.add_vector(&zero_vector).unwrap();

        vector_storage.sync().unwrap();

        let config = ShardexConfig::new()
            .directory_path(temp_dir.path().to_path_buf())
            .vector_size(4);

        let mut checker = IntegrityChecker::new(temp_dir.path().to_path_buf(), config);

        // Test vector storage file validation
        let result = checker
            .verify_vector_storage_file(&vector_path)
            .await
            .unwrap();

        // Should pass because we don't have excessive problematic vectors
        assert!(result.is_valid());
    }

    #[test]
    fn test_component_type_name() {
        assert_eq!(component_type_name(ComponentType::VectorStorage), "VectorStorage");
        assert_eq!(component_type_name(ComponentType::PostingStorage), "PostingStorage");
        assert_eq!(component_type_name(ComponentType::WalSegments), "WalSegments");
        assert_eq!(component_type_name(ComponentType::BloomFilters), "BloomFilters");
        assert_eq!(component_type_name(ComponentType::ShardexSegments), "ShardexSegments");
        assert_eq!(component_type_name(ComponentType::CrossReferences), "CrossReferences");
    }

    #[tokio::test]
    async fn test_repair_strategies() {
        let temp_dir = TempDir::new().unwrap();
        let config = ShardexConfig::new()
            .directory_path(temp_dir.path().to_path_buf())
            .vector_size(128);

        let mut checker = IntegrityChecker::new(temp_dir.path().to_path_buf(), config);

        // Test NaN repair strategy
        let nan_corruption = CorruptionReport {
            corruption_type: CorruptionType::DataCorruption,
            file_path: temp_dir.path().join("test_file"),
            corruption_offset: Some(100),
            corruption_size: Some(50),
            description: "Excessive NaN values in vectors: 15.0% (3/20)".to_string(),
            recovery_recommendations: vec!["Check vector computation logic".to_string()],
            severity: 0.8,
            is_recoverable: true,
            detected_at: SystemTime::now(),
        };

        let (success, action, notes) = checker
            .repair_vector_quality_issues(&nan_corruption)
            .await
            .unwrap();
        assert!(!success); // Should not succeed as it's not fully implemented
        assert!(action.contains("NaN repair"));
        assert!(!notes.is_empty());

        // Test count mismatch repair strategy
        let count_mismatch = CorruptionReport {
            corruption_type: CorruptionType::CrossValidationFailure,
            file_path: temp_dir.path().join("test_file"),
            corruption_offset: None,
            corruption_size: None,
            description: "Count mismatch: posting storage has 5, vector storage has 3".to_string(),
            recovery_recommendations: vec!["Run consistency repair".to_string()],
            severity: 0.7,
            is_recoverable: true,
            detected_at: SystemTime::now(),
        };

        let (success, action, notes) = checker
            .repair_count_mismatch(&count_mismatch)
            .await
            .unwrap();
        assert!(success); // Low severity count mismatches can be repaired
        assert!(action.contains("can be automatically repaired"));
        assert!(!notes.is_empty());
    }

    #[tokio::test]
    async fn test_capacity_consistency_validation() {
        let temp_dir = TempDir::new().unwrap();

        // Create posting storage with capacity 10
        let posting_path = temp_dir.path().join("test_shard.postings");
        let _posting_storage = crate::posting_storage::PostingStorage::create(&posting_path, 10).unwrap();

        // Create vector storage with different capacity
        let vector_path = temp_dir.path().join("test_shard.vectors");
        let _vector_storage = crate::vector_storage::VectorStorage::create(&vector_path, 128, 5).unwrap(); // Different capacity

        let config = ShardexConfig::new()
            .directory_path(temp_dir.path().to_path_buf())
            .vector_size(128);

        let checker = IntegrityChecker::new(temp_dir.path().to_path_buf(), config);

        // Test consistency validation
        let posting_mmf = MemoryMappedFile::open_read_only(&posting_path).unwrap();
        let vector_mmf = MemoryMappedFile::open_read_only(&vector_path).unwrap();

        let result = checker
            .verify_header_compatibility(&posting_mmf, &vector_mmf)
            .await;
        assert!(result.is_err());

        let error = result.unwrap_err();
        assert_eq!(error.corruption_type, CorruptionType::CrossValidationFailure);
        assert!(error.description.contains("Capacity mismatch"));
    }

    #[test]
    fn test_detailed_corruption_analysis() {
        let temp_dir = TempDir::new().unwrap();
        let file_path = temp_dir.path().join("detailed_analysis.dat");

        // Create file with specific corruption pattern
        let mut mmf = MemoryMappedFile::create(&file_path, 1024).unwrap();
        let data_size = 1024 - FileHeader::SIZE; // All remaining space after header
        let mut test_data = vec![42u8; data_size];

        // Corrupt specific bytes in the middle
        test_data[50] = 0xFF;
        test_data[51] = 0xFF;

        let header = FileHeader::new(magic::TEST_GENERIC, 1, FileHeader::SIZE as u64, &vec![42u8; data_size]); // Original checksum

        mmf.write_at(0, &header).unwrap();
        mmf.write_slice_at(FileHeader::SIZE, &test_data).unwrap(); // Corrupted data
        mmf.sync().unwrap();
        drop(mmf);

        let config = IntegrityConfig {
            detailed_analysis: true,
            ..Default::default()
        };

        let mut manager = IntegrityManager::new(config);
        let result = manager.validate_file_path(&file_path).unwrap();

        assert!(!result.is_valid());
        let report = result.corruption_report().unwrap();

        // Verify detailed corruption report
        assert!(report.description.contains("Checksum mismatch")); // Note: Capital C
        assert!(!report.recovery_recommendations.is_empty());
        assert_eq!(report.corruption_type, CorruptionType::DataCorruption);
        assert!(report.severity > 0.0);
    }

    #[test]
    fn test_verify_posting_data_quality_valid_data() {
        let temp_dir = TempDir::new().unwrap();
        let config = ShardexConfig::default();
        let checker = IntegrityChecker::new(temp_dir.path().to_path_buf(), config);

        let header_size = std::mem::size_of::<crate::posting_storage::PostingStorageHeader>();

        // Create a valid posting storage header
        let header = crate::posting_storage::PostingStorageHeader {
            file_header: crate::memory::FileHeader::new_without_checksum(magic::POSTING_STORAGE, 1, 128),
            capacity: 2,
            current_count: 2,
            active_count: 2,
            document_ids_offset: header_size as u64, // Start right after header
            starts_offset: (header_size + 2 * 16) as u64, // After document IDs
            lengths_offset: (header_size + 2 * 16 + 2 * 4) as u64, // After starts
            deleted_flags_offset: (header_size + 2 * 16 + 2 * 4 + 2 * 4) as u64, // After lengths
            document_id_size: 16,
            reserved: [0; 12],
        };

        // Create valid posting data sized properly for the data structure
        let mut posting_data = vec![0u8; 2 * 16 + 2 * 4 + 2 * 4 + 1]; // Document IDs + starts + lengths + deleted flags

        // Add two valid document IDs in ascending order
        use crate::identifiers::DocumentId;
        let doc_id1 = DocumentId::from_raw(1000000000000000000u128);
        let doc_id2 = DocumentId::from_raw(2000000000000000000u128);

        posting_data[0..16].copy_from_slice(&doc_id1.to_bytes());
        posting_data[16..32].copy_from_slice(&doc_id2.to_bytes());

        // Add valid start positions
        let starts_offset = 2 * 16;
        posting_data[starts_offset..starts_offset + 4].copy_from_slice(&100u32.to_le_bytes());
        posting_data[starts_offset + 4..starts_offset + 8].copy_from_slice(&200u32.to_le_bytes());

        // Add valid lengths
        let lengths_offset = 2 * 16 + 2 * 4;
        posting_data[lengths_offset..lengths_offset + 4].copy_from_slice(&50u32.to_le_bytes());
        posting_data[lengths_offset + 4..lengths_offset + 8].copy_from_slice(&75u32.to_le_bytes());

        // Add deleted flags (both not deleted)
        let deleted_flags_offset = 2 * 16 + 2 * 4 + 2 * 4;
        posting_data[deleted_flags_offset] = 0b00000000;

        let result = checker.verify_posting_data_quality(&posting_data, &header);
        assert!(
            result.is_ok(),
            "Expected valid data to pass verification, but got error: {:?}",
            result.err()
        );
    }

    #[test]
    fn test_verify_posting_data_quality_empty_storage() {
        let temp_dir = TempDir::new().unwrap();
        let config = ShardexConfig::default();
        let checker = IntegrityChecker::new(temp_dir.path().to_path_buf(), config);

        let header = crate::posting_storage::PostingStorageHeader {
            file_header: crate::memory::FileHeader::new_without_checksum(magic::POSTING_STORAGE, 1, 64),
            capacity: 10,
            current_count: 0,
            active_count: 0,
            document_ids_offset: 64,
            starts_offset: 64,
            lengths_offset: 64,
            deleted_flags_offset: 64,
            document_id_size: 16,
            reserved: [0; 12],
        };

        let posting_data = vec![0u8; 64];
        let result = checker.verify_posting_data_quality(&posting_data, &header);
        assert!(result.is_ok());
    }

    #[test]
    fn test_verify_posting_data_quality_count_exceeds_capacity() {
        let temp_dir = TempDir::new().unwrap();
        let config = ShardexConfig::default();
        let checker = IntegrityChecker::new(temp_dir.path().to_path_buf(), config);

        let header = crate::posting_storage::PostingStorageHeader {
            file_header: crate::memory::FileHeader::new_without_checksum(magic::POSTING_STORAGE, 1, 320),
            capacity: 5,
            current_count: 10, // Exceeds capacity
            active_count: 10,
            document_ids_offset: 64,
            starts_offset: 224,
            lengths_offset: 264,
            deleted_flags_offset: 304,
            document_id_size: 16,
            reserved: [0; 12],
        };

        let posting_data = vec![0u8; 320];
        let result = checker.verify_posting_data_quality(&posting_data, &header);

        assert!(result.is_err());
        let error = result.unwrap_err();
        assert!(error.description.contains("Current count"));
        assert!(error.description.contains("exceeds capacity"));
        assert_eq!(error.severity, 0.9);
        assert!(error.is_recoverable);
    }

    #[test]
    fn test_verify_posting_data_quality_arrays_beyond_bounds() {
        let temp_dir = TempDir::new().unwrap();
        let config = ShardexConfig::default();
        let checker = IntegrityChecker::new(temp_dir.path().to_path_buf(), config);

        let header = crate::posting_storage::PostingStorageHeader {
            file_header: crate::memory::FileHeader::new_without_checksum(magic::POSTING_STORAGE, 1, 120),
            capacity: 10,
            current_count: 2,
            active_count: 2,
            document_ids_offset: 144,  // Must be >= header size (140 bytes)
            starts_offset: 176,        // 144 + (2 * 16) = 176
            lengths_offset: 184,       // 176 + (2 * 4) = 184
            deleted_flags_offset: 500, // Way beyond file size (this should cause the bounds error)
            document_id_size: 16,
            reserved: [0; 12],
        };

        let posting_data = vec![0u8; 200]; // Large enough for valid offsets but not for deleted_flags_offset
        let result = checker.verify_posting_data_quality(&posting_data, &header);

        assert!(result.is_err());
        let error = result.unwrap_err();
        assert!(error.description.contains("extend beyond file bounds"));
        assert_eq!(error.severity, 1.0);
        assert!(!error.is_recoverable);
    }

    #[test]
    fn test_verify_posting_data_quality_too_many_invalid_document_ids() {
        let temp_dir = TempDir::new().unwrap();
        let config = ShardexConfig::default();
        let checker = IntegrityChecker::new(temp_dir.path().to_path_buf(), config);

        let header = crate::posting_storage::PostingStorageHeader {
            file_header: crate::memory::FileHeader::new_without_checksum(magic::POSTING_STORAGE, 1, 200),
            capacity: 10,
            current_count: 5,
            active_count: 5,
            document_ids_offset: 144,  // Must be >= header size (140 bytes)
            starts_offset: 224,        // 144 + (5 * 16) = 224
            lengths_offset: 244,       // 224 + (5 * 4) = 244
            deleted_flags_offset: 264, // 244 + (5 * 4) = 264
            document_id_size: 16,
            reserved: [0; 12],
        };

        let mut posting_data = vec![0u8; 300];

        // Add mostly invalid document IDs (all zeros)
        // Only make one valid to ensure > 10% are invalid
        let valid_doc_id = DocumentId::new().raw().to_le_bytes();
        posting_data[144..160].copy_from_slice(&valid_doc_id);
        // The rest remain zero (invalid)

        // Add valid start positions and lengths
        for i in 0..5 {
            posting_data[224 + i * 4..228 + i * 4].copy_from_slice(&(100u32 + i as u32 * 50).to_le_bytes());
            posting_data[244 + i * 4..248 + i * 4].copy_from_slice(&50u32.to_le_bytes());
        }

        // No deleted flags
        posting_data[264] = 0b00000000;

        let result = checker.verify_posting_data_quality(&posting_data, &header);

        assert!(result.is_err());
        let error = result.unwrap_err();
        assert!(error.description.contains("Too many invalid document IDs"));
        assert_eq!(error.severity, 0.8);
        assert!(error.is_recoverable);
    }

    #[test]
    fn test_verify_posting_data_quality_too_many_zero_lengths() {
        let temp_dir = TempDir::new().unwrap();
        let config = ShardexConfig::default();
        let checker = IntegrityChecker::new(temp_dir.path().to_path_buf(), config);

        let header_size = std::mem::size_of::<crate::posting_storage::PostingStorageHeader>();

        let header = crate::posting_storage::PostingStorageHeader {
            file_header: crate::memory::FileHeader::new_without_checksum(magic::POSTING_STORAGE, 1, 320),
            capacity: 10,
            current_count: 10,
            active_count: 10,
            document_ids_offset: header_size as u64, // Start right after header
            starts_offset: (header_size + 10 * 16) as u64, // After document IDs
            lengths_offset: (header_size + 10 * 16 + 10 * 4) as u64, // After starts
            deleted_flags_offset: (header_size + 10 * 16 + 10 * 4 + 10 * 4) as u64, // After lengths
            document_id_size: 16,
            reserved: [0; 12],
        };

        let data_size = 10 * 16 + 10 * 4 + 10 * 4 + 2;
        let mut posting_data = vec![0u8; data_size]; // Document IDs + starts + lengths + deleted flags

        // Add valid document IDs in ascending order to avoid ordering violations
        for i in 0..10 {
            let doc_id = DocumentId::from_raw(1000u128 + i as u128)
                .raw()
                .to_le_bytes();
            posting_data[i * 16..(i + 1) * 16].copy_from_slice(&doc_id);
        }

        // Add valid start positions
        let starts_offset = 10 * 16;
        for i in 0..10 {
            posting_data[starts_offset + i * 4..starts_offset + (i + 1) * 4]
                .copy_from_slice(&(100u32 + i as u32 * 10).to_le_bytes());
        }

        // Add mostly zero lengths (more than 5% threshold) - 8 out of 10 = 80%
        let lengths_offset = 10 * 16 + 10 * 4;
        for i in 0..8 {
            posting_data[lengths_offset + i * 4..lengths_offset + (i + 1) * 4].copy_from_slice(&0u32.to_le_bytes());
            // Zero length
        }
        // Add a few valid lengths
        posting_data[lengths_offset + 8 * 4..lengths_offset + 9 * 4].copy_from_slice(&50u32.to_le_bytes());
        posting_data[lengths_offset + 9 * 4..lengths_offset + 10 * 4].copy_from_slice(&75u32.to_le_bytes());

        // No deleted flags
        let deleted_flags_offset = 10 * 16 + 10 * 4 + 10 * 4;
        posting_data[deleted_flags_offset] = 0b00000000;
        posting_data[deleted_flags_offset + 1] = 0b00000000;

        let result = checker.verify_posting_data_quality(&posting_data, &header);

        assert!(result.is_err(), "Expected error but got success");
        let error = result.unwrap_err();
        assert!(error.description.contains("Too many zero-length postings:"));
        assert_eq!(error.severity, 0.4);
        assert!(error.is_recoverable);
    }

    #[test]
    fn test_verify_posting_data_quality_overflow_ranges() {
        let temp_dir = TempDir::new().unwrap();
        let config = ShardexConfig::default();
        let checker = IntegrityChecker::new(temp_dir.path().to_path_buf(), config);

        let header = crate::posting_storage::PostingStorageHeader {
            file_header: crate::memory::FileHeader::new_without_checksum(magic::POSTING_STORAGE, 1, 120),
            capacity: 10,
            current_count: 2,
            active_count: 2,
            document_ids_offset: 144,  // Must be >= header size (140 bytes)
            starts_offset: 176,        // 144 + (2 * 16) = 176
            lengths_offset: 184,       // 176 + (2 * 4) = 184
            deleted_flags_offset: 192, // 184 + (2 * 4) = 192
            document_id_size: 16,
            reserved: [0; 12],
        };

        let mut posting_data = vec![0u8; 200];

        // Add valid document IDs
        let doc_id1 = DocumentId::new().raw().to_le_bytes();
        let doc_id2 = DocumentId::new().raw().to_le_bytes();
        posting_data[144..160].copy_from_slice(&doc_id1);
        posting_data[160..176].copy_from_slice(&doc_id2);

        // Add start positions that will overflow when added to lengths
        posting_data[176..180].copy_from_slice(&(u32::MAX - 10).to_le_bytes());
        posting_data[180..184].copy_from_slice(&100u32.to_le_bytes());

        // Add lengths that cause overflow
        posting_data[184..188].copy_from_slice(&20u32.to_le_bytes()); // This will overflow with start
        posting_data[188..192].copy_from_slice(&50u32.to_le_bytes());

        // No deleted flags
        posting_data[192] = 0b00000000;

        let result = checker.verify_posting_data_quality(&posting_data, &header);

        // The test should detect some form of data corruption
        assert!(result.is_err());
        let error = result.unwrap_err();
        assert_eq!(error.corruption_type, CorruptionType::DataCorruption);
        assert!(error.is_recoverable);
    }

    #[test]
    fn test_verify_posting_data_quality_active_count_mismatch() {
        let temp_dir = TempDir::new().unwrap();
        let config = ShardexConfig::default();
        let checker = IntegrityChecker::new(temp_dir.path().to_path_buf(), config);

        let header = crate::posting_storage::PostingStorageHeader {
            file_header: crate::memory::FileHeader::new_without_checksum(magic::POSTING_STORAGE, 1, 140),
            capacity: 10,
            current_count: 3,
            active_count: 2,           // Claims 2 active but we'll make all 3 active
            document_ids_offset: 144,  // Must be >= header size (140 bytes)
            starts_offset: 192,        // 144 + (3 * 16) = 192
            lengths_offset: 204,       // 192 + (3 * 4) = 204
            deleted_flags_offset: 216, // 204 + (3 * 4) = 216
            document_id_size: 16,
            reserved: [0; 12],
        };

        let mut posting_data = vec![0u8; 220];

        // Add valid document IDs
        for i in 0..3 {
            let doc_id = DocumentId::new().raw().to_le_bytes();
            posting_data[144 + i * 16..160 + i * 16].copy_from_slice(&doc_id);
        }

        // Add valid start positions and lengths
        for i in 0..3 {
            posting_data[192 + i * 4..196 + i * 4].copy_from_slice(&(100u32 + i as u32 * 50).to_le_bytes());
            posting_data[204 + i * 4..208 + i * 4].copy_from_slice(&50u32.to_le_bytes());
        }

        // All postings are active (no deleted flags set)
        posting_data[216] = 0b00000000; // All bits 0 means all active

        let result = checker.verify_posting_data_quality(&posting_data, &header);

        assert!(result.is_err());
        let error = result.unwrap_err();
        assert!(error.description.contains("Active count mismatch"));
        assert!(error.description.contains("header claims 2, actual 3"));
        assert_eq!(error.severity, 0.6);
        assert!(error.is_recoverable);
    }

    #[test]
    fn test_overflow_arithmetic() {
        let start = u32::MAX - 10;
        let length = 20u32;
        assert!(
            start.checked_add(length).is_none(),
            "Expected overflow but got {:?}",
            start.checked_add(length)
        );
    }

    #[test]
    fn test_recovery_header_corruption() {
        let temp_dir = TempDir::new().unwrap();
        let config = IntegrityConfig {
            enable_recovery: true,
            ..IntegrityConfig::default()
        };
        let mut manager = IntegrityManager::new(config);

        let corruption_report = CorruptionReport {
            corruption_type: CorruptionType::HeaderCorruption,
            file_path: temp_dir.path().join("test_vector.vec"),
            corruption_offset: Some(0),
            corruption_size: Some(FileHeader::SIZE as u64),
            description: "magic bytes corrupted".to_string(),
            recovery_recommendations: vec!["Restore from backup".to_string()],
            severity: 0.9,
            is_recoverable: true,
            detected_at: SystemTime::now(),
        };

        // Test the recovery through the public interface
        let result = manager.attempt_recovery(&corruption_report).unwrap();

        // Header corruption recovery should be limited
        assert!(!result); // Should not be successful for header corruption
    }

    #[test]
    fn test_recovery_file_truncation() {
        let temp_dir = TempDir::new().unwrap();
        let config = IntegrityConfig {
            enable_recovery: true,
            ..IntegrityConfig::default()
        };
        let mut manager = IntegrityManager::new(config);

        // Test minor truncation (recoverable)
        let minor_corruption = CorruptionReport {
            corruption_type: CorruptionType::FileTruncation,
            file_path: temp_dir.path().join("test.dat"),
            corruption_offset: Some(4096), // Page boundary
            corruption_size: Some(1024),
            description: "File truncated at offset 4096".to_string(),
            recovery_recommendations: vec!["Extend file".to_string()],
            severity: 0.3, // Minor
            is_recoverable: true,
            detected_at: SystemTime::now(),
        };

        let result = manager.attempt_recovery(&minor_corruption).unwrap();
        assert!(result); // Should be recoverable

        // Test severe truncation (not recoverable)
        let mut manager2 = IntegrityManager::new(IntegrityConfig {
            enable_recovery: true,
            ..IntegrityConfig::default()
        });
        let severe_corruption = CorruptionReport {
            corruption_type: CorruptionType::FileTruncation,
            file_path: temp_dir.path().join("test2.dat"),
            corruption_offset: Some(100),
            corruption_size: Some(1000000),
            description: "Severe file truncation".to_string(),
            recovery_recommendations: vec![],
            severity: 0.9, // Severe
            is_recoverable: true,
            detected_at: SystemTime::now(),
        };

        let result2 = manager2.attempt_recovery(&severe_corruption).unwrap();
        assert!(!result2); // Should not be recoverable due to severity
    }

    #[test]
    fn test_recovery_structural_inconsistency() {
        let temp_dir = TempDir::new().unwrap();
        let config = IntegrityConfig {
            enable_recovery: true,
            ..IntegrityConfig::default()
        };
        let mut manager = IntegrityManager::new(config);

        let corruption_report = CorruptionReport {
            corruption_type: CorruptionType::StructuralInconsistency,
            file_path: temp_dir.path().join("test_index.idx"),
            corruption_offset: Some(1000),
            corruption_size: Some(256),
            description: "index structure corruption detected".to_string(),
            recovery_recommendations: vec!["Rebuild index".to_string()],
            severity: 0.6,
            is_recoverable: true,
            detected_at: SystemTime::now(),
        };

        let result = manager.attempt_recovery(&corruption_report).unwrap();
        assert!(result); // Index structures are rebuildable
    }

    #[test]
    fn test_attempt_recovery_all_types() {
        let config = IntegrityConfig {
            enable_recovery: true,
            ..IntegrityConfig::default()
        };
        let mut manager = IntegrityManager::new(config);

        // Test recovery for each corruption type
        let corruption_types = vec![
            CorruptionType::DataCorruption,
            CorruptionType::HeaderCorruption,
            CorruptionType::FileTruncation,
            CorruptionType::StructuralInconsistency,
            CorruptionType::CrossValidationFailure,
            CorruptionType::PartialCorruption,
        ];

        for corruption_type in corruption_types {
            let corruption_report = CorruptionReport {
                corruption_type: corruption_type.clone(),
                file_path: PathBuf::from("/test/file.dat"),
                corruption_offset: Some(1000),
                corruption_size: Some(100),
                description: format!("Test {} corruption", format!("{:?}", corruption_type).to_lowercase()),
                recovery_recommendations: vec![],
                severity: 0.5,
                is_recoverable: true,
                detected_at: SystemTime::now(),
            };

            let result = manager.attempt_recovery(&corruption_report);
            assert!(result.is_ok(), "Recovery attempt failed for {:?}", corruption_type);

            // The result (success/failure) depends on the specific corruption type and conditions
            // but the function should not panic or return errors
        }
    }

    #[test]
    fn test_recovery_data_corruption() {
        // This test verifies the logical flow of data corruption recovery
        let temp_dir = TempDir::new().unwrap();
        let config = IntegrityConfig {
            enable_recovery: true,
            ..IntegrityConfig::default()
        };
        let mut manager = IntegrityManager::new(config);

        let corruption_report = CorruptionReport {
            corruption_type: CorruptionType::DataCorruption,
            file_path: temp_dir.path().join("test.dat"),
            corruption_offset: Some(100),
            corruption_size: Some(50),
            description: "Small data corruption detected".to_string(),
            recovery_recommendations: vec!["Attempt data recovery".to_string()],
            severity: 0.4, // Minor
            is_recoverable: true,
            detected_at: SystemTime::now(),
        };

        // Small corruption should be recoverable
        let result = manager.attempt_recovery(&corruption_report).unwrap();
        assert!(result); // Should indicate recovery is possible
    }

    #[test]
    fn test_recovery_with_disabled_config() {
        let config = IntegrityConfig {
            enable_recovery: false, // Disabled
            ..IntegrityConfig::default()
        };
        let mut manager = IntegrityManager::new(config);

        let corruption_report = CorruptionReport {
            corruption_type: CorruptionType::DataCorruption,
            file_path: PathBuf::from("/test/file.dat"),
            corruption_offset: Some(100),
            corruption_size: Some(50),
            description: "Test corruption".to_string(),
            recovery_recommendations: vec![],
            severity: 0.5,
            is_recoverable: true,
            detected_at: SystemTime::now(),
        };

        let result = manager.attempt_recovery(&corruption_report);
        assert!(result.is_ok());
        assert!(!result.unwrap()); // Should return false when recovery is disabled
    }

    #[tokio::test]
    async fn test_pointer_corruption_repair_vector_storage() {
        let temp_dir = TempDir::new().unwrap();
        let config = ShardexConfig::new()
            .directory_path(temp_dir.path().to_path_buf())
            .vector_size(128);

        let mut checker = IntegrityChecker::new(temp_dir.path().to_path_buf(), config);

        // Create a vector storage file with valid content
        let vector_path = temp_dir.path().join("test_vectors.vectors");
        let _vector_storage = crate::vector_storage::VectorStorage::create(&vector_path, 128, 10).unwrap();

        // Create corruption report for pointer/offset issues
        let corruption = CorruptionReport {
            corruption_type: CorruptionType::StructuralInconsistency,
            file_path: vector_path,
            corruption_offset: Some(100),
            corruption_size: Some(8),
            description: "pointer corruption in vector storage detected".to_string(),
            recovery_recommendations: vec!["Validate and repair vector offsets".to_string()],
            severity: 0.6,
            is_recoverable: true,
            detected_at: SystemTime::now(),
        };

        // Test pointer validation and repair
        let result = checker.repair_structural_inconsistency(&corruption).await;
        assert!(result.is_ok());

        let (_success, message, notes) = result.unwrap();

        // Should attempt repair but may not succeed depending on actual corruption
        assert!(!notes.is_empty());
        assert!(message.contains("Pointer") || message.contains("pointer"));
        assert!(notes
            .iter()
            .any(|note| note.contains("Validating pointers")));
    }

    #[tokio::test]
    async fn test_pointer_corruption_repair_posting_storage() {
        let temp_dir = TempDir::new().unwrap();
        let config = ShardexConfig::new()
            .directory_path(temp_dir.path().to_path_buf())
            .vector_size(128);

        let mut checker = IntegrityChecker::new(temp_dir.path().to_path_buf(), config);

        // Create a posting storage file
        let posting_path = temp_dir.path().join("test_postings.postings");
        let _posting_storage = crate::posting_storage::PostingStorage::create(&posting_path, 10).unwrap();

        let corruption = CorruptionReport {
            corruption_type: CorruptionType::StructuralInconsistency,
            file_path: posting_path,
            corruption_offset: Some(200),
            corruption_size: Some(8),
            description: "offset corruption in posting storage detected".to_string(),
            recovery_recommendations: vec!["Repair vector offset pointers".to_string()],
            severity: 0.7,
            is_recoverable: true,
            detected_at: SystemTime::now(),
        };

        let result = checker.repair_structural_inconsistency(&corruption).await;
        assert!(result.is_ok());

        let (_success, _message, notes) = result.unwrap();

        // Should detect and attempt to repair posting storage pointers
        assert!(!notes.is_empty());
        assert!(notes
            .iter()
            .any(|note| note.contains("posting") || note.contains("Posting")));
    }

    #[tokio::test]
    async fn test_pointer_validation_invalid_magic_bytes() {
        let temp_dir = TempDir::new().unwrap();
        let config = ShardexConfig::new()
            .directory_path(temp_dir.path().to_path_buf())
            .vector_size(128);

        let _checker = IntegrityChecker::new(temp_dir.path().to_path_buf(), config);

        // Create a file with invalid magic bytes
        let test_file = temp_dir.path().join("invalid_magic.dat");
        let mut mmf = MemoryMappedFile::create(&test_file, 1024).unwrap();

        // Write invalid header with corrupted magic bytes
        let invalid_header = crate::memory::StandardHeader::new_without_checksum(
            b"XXXX", // Invalid magic
            1,
            crate::memory::StandardHeader::SIZE as u64,
        );
        mmf.write_at(0, &invalid_header).unwrap();
        mmf.sync().unwrap();
        drop(mmf);

        let corruption = CorruptionReport {
            corruption_type: CorruptionType::StructuralInconsistency,
            file_path: test_file,
            corruption_offset: Some(0),
            corruption_size: Some(4),
            description: "pointer corruption with invalid magic bytes".to_string(),
            recovery_recommendations: vec!["Fix magic bytes".to_string()],
            severity: 0.8,
            is_recoverable: true,
            detected_at: SystemTime::now(),
        };

        // Create a mutable checker for the repair
        let mut checker_mut = IntegrityChecker::new(
            temp_dir.path().to_path_buf(),
            ShardexConfig::new()
                .directory_path(temp_dir.path().to_path_buf())
                .vector_size(128),
        );

        let result = checker_mut
            .repair_structural_inconsistency(&corruption)
            .await;
        assert!(result.is_ok());

        let (_success, _message, notes) = result.unwrap();

        // Should detect invalid magic bytes
        assert!(!notes.is_empty());
        assert!(notes
            .iter()
            .any(|note| note.contains("magic") || note.contains("Invalid")));
    }

    #[tokio::test]
    async fn test_pointer_validation_data_offset_out_of_bounds() {
        let temp_dir = TempDir::new().unwrap();
        let config = ShardexConfig::new()
            .directory_path(temp_dir.path().to_path_buf())
            .vector_size(128);

        let _checker = IntegrityChecker::new(temp_dir.path().to_path_buf(), config);

        // Create a file with data_offset pointing beyond file size
        let test_file = temp_dir.path().join("out_of_bounds.dat");
        let file_size = 512u64;
        let mut mmf = MemoryMappedFile::create(&test_file, file_size as usize).unwrap();

        // Write header with data_offset beyond file size
        let bad_header = crate::memory::StandardHeader::new_without_checksum(
            crate::constants::magic::TEST_GENERIC,
            1,
            file_size + 1000, // Beyond file size
        );
        mmf.write_at(0, &bad_header).unwrap();
        mmf.sync().unwrap();
        drop(mmf);

        let corruption = CorruptionReport {
            corruption_type: CorruptionType::StructuralInconsistency,
            file_path: test_file,
            corruption_offset: Some(16), // data_offset field location
            corruption_size: Some(8),
            description: "pointer data_offset exceeds file bounds".to_string(),
            recovery_recommendations: vec!["Fix data offset pointer".to_string()],
            severity: 0.9,
            is_recoverable: true,
            detected_at: SystemTime::now(),
        };

        let mut checker_mut = IntegrityChecker::new(
            temp_dir.path().to_path_buf(),
            ShardexConfig::new()
                .directory_path(temp_dir.path().to_path_buf())
                .vector_size(128),
        );

        let result = checker_mut
            .repair_structural_inconsistency(&corruption)
            .await;
        assert!(result.is_ok());

        let (_success, _message, notes) = result.unwrap();

        // Should detect data_offset out of bounds
        assert!(!notes.is_empty());
        assert!(notes
            .iter()
            .any(|note| note.contains("exceeds file size") || note.contains("out of bounds")));
    }

    #[tokio::test]
    async fn test_pointer_validation_alignment_issues() {
        let temp_dir = TempDir::new().unwrap();
        let config = ShardexConfig::new()
            .directory_path(temp_dir.path().to_path_buf())
            .vector_size(128);

        let _checker = IntegrityChecker::new(temp_dir.path().to_path_buf(), config);

        // Create a file with misaligned data_offset
        let test_file = temp_dir.path().join("misaligned.dat");
        let mut mmf = MemoryMappedFile::create(&test_file, 1024).unwrap();

        // Write header with misaligned data_offset (not 8-byte aligned)
        let misaligned_header = crate::memory::StandardHeader::new_without_checksum(
            crate::constants::magic::TEST_GENERIC,
            1,
            crate::memory::StandardHeader::SIZE as u64 + 3, // Misaligned by 3 bytes
        );
        mmf.write_at(0, &misaligned_header).unwrap();
        mmf.sync().unwrap();
        drop(mmf);

        let corruption = CorruptionReport {
            corruption_type: CorruptionType::StructuralInconsistency,
            file_path: test_file,
            corruption_offset: Some(16),
            corruption_size: Some(8),
            description: "pointer alignment corruption detected".to_string(),
            recovery_recommendations: vec!["Fix pointer alignment".to_string()],
            severity: 0.5,
            is_recoverable: true,
            detected_at: SystemTime::now(),
        };

        let mut checker_mut = IntegrityChecker::new(
            temp_dir.path().to_path_buf(),
            ShardexConfig::new()
                .directory_path(temp_dir.path().to_path_buf())
                .vector_size(128),
        );

        let result = checker_mut
            .repair_structural_inconsistency(&corruption)
            .await;
        assert!(result.is_ok());

        let (_success, _message, notes) = result.unwrap();

        // Should detect alignment issues
        assert!(!notes.is_empty());
        assert!(notes
            .iter()
            .any(|note| note.contains("aligned") || note.contains("alignment")));
    }

    #[tokio::test]
    async fn test_pointer_validation_wal_file() {
        let temp_dir = TempDir::new().unwrap();
        let config = ShardexConfig::new()
            .directory_path(temp_dir.path().to_path_buf())
            .vector_size(128);

        let _checker = IntegrityChecker::new(temp_dir.path().to_path_buf(), config);

        // Create a WAL file
        let wal_file = temp_dir.path().join("test.wal");
        let mut mmf = MemoryMappedFile::create(&wal_file, 1024).unwrap();

        // Write valid WAL header
        let wal_header = crate::memory::StandardHeader::new_without_checksum(
            crate::constants::magic::WAL,
            1,
            crate::memory::StandardHeader::SIZE as u64,
        );
        mmf.write_at(0, &wal_header).unwrap();
        mmf.sync().unwrap();
        drop(mmf);

        let corruption = CorruptionReport {
            corruption_type: CorruptionType::StructuralInconsistency,
            file_path: wal_file,
            corruption_offset: Some(100),
            corruption_size: Some(8),
            description: "pointer issues in WAL file detected".to_string(),
            recovery_recommendations: vec!["Validate WAL pointers".to_string()],
            severity: 0.4,
            is_recoverable: true,
            detected_at: SystemTime::now(),
        };

        let mut checker_mut = IntegrityChecker::new(
            temp_dir.path().to_path_buf(),
            ShardexConfig::new()
                .directory_path(temp_dir.path().to_path_buf())
                .vector_size(128),
        );

        let result = checker_mut
            .repair_structural_inconsistency(&corruption)
            .await;
        assert!(result.is_ok());

        let (_success, _message, notes) = result.unwrap();

        // Should validate WAL file structure
        assert!(!notes.is_empty());
        assert!(notes
            .iter()
            .any(|note| note.contains("WAL") || note.contains("wal")));
    }

    #[test]
    fn test_is_valid_magic() {
        let temp_dir = TempDir::new().unwrap();
        let config = ShardexConfig::new()
            .directory_path(temp_dir.path().to_path_buf())
            .vector_size(128);

        let checker = IntegrityChecker::new(temp_dir.path().to_path_buf(), config);

        // Test valid magic bytes
        assert!(checker.is_valid_magic(crate::constants::magic::WAL));
        assert!(checker.is_valid_magic(crate::constants::magic::VECTOR_STORAGE));
        assert!(checker.is_valid_magic(crate::constants::magic::POSTING_STORAGE));
        assert!(checker.is_valid_magic(crate::constants::magic::TEXT_INDEX));
        assert!(checker.is_valid_magic(crate::constants::magic::TEXT_DATA));

        // Test invalid magic bytes
        assert!(!checker.is_valid_magic(b"XXXX"));
        assert!(!checker.is_valid_magic(b"YYYY"));
        assert!(!checker.is_valid_magic(b"ZZZZ"));

        // Test test magic bytes (should be invalid for production validation)
        assert!(!checker.is_valid_magic(crate::constants::magic::TEST_GENERIC));
        assert!(!checker.is_valid_magic(crate::constants::magic::TEST_SHARD));
    }

    #[tokio::test]
    async fn test_repair_vector_data_offset_successful() {
        use crate::memory::MemoryMappedFile;
        use crate::vector_storage::{VectorStorage, VectorStorageHeader};
        use tempfile::TempDir;

        let temp_dir = TempDir::new().unwrap();
        let vector_file = temp_dir.path().join("test_vector.dat");

        // Create a vector storage file with valid content
        let storage = VectorStorage::create(&vector_file, 4, 100).unwrap();
        drop(storage); // Close file to allow repair operations

        // Now corrupt the vector_data_offset by setting it to an invalid value
        {
            let mut mmf = MemoryMappedFile::open_read_write(&vector_file).unwrap();
            let mut header: VectorStorageHeader = mmf.read_at(0).unwrap();
            let _original_offset = header.vector_data_offset;

            // Set invalid offset that exceeds file size
            header.vector_data_offset = mmf.len() as u64 + 1000;
            mmf.write_at(0, &header).unwrap();

            // Verify corruption was written
            let corrupted_header: VectorStorageHeader = mmf.read_at(0).unwrap();
            assert_eq!(corrupted_header.vector_data_offset, mmf.len() as u64 + 1000);
        }

        // Now test repair functionality
        let config = ShardexConfig::new()
            .directory_path(temp_dir.path().to_path_buf())
            .vector_size(4);
        let checker = IntegrityChecker::new(temp_dir.path().to_path_buf(), config);

        // Calculate a reasonable new offset
        let reasonable_offset = std::mem::size_of::<crate::memory::StandardHeader>() as u64
            + std::mem::size_of::<VectorStorageHeader>() as u64;

        let mut notes = Vec::new();
        let result = checker
            .repair_vector_data_offset(&vector_file, reasonable_offset, &mut notes)
            .await;

        // Verify repair succeeded
        assert!(result.is_ok());
        assert!(result.unwrap()); // Should return true for successful repair

        // Verify repair was actually applied
        {
            let mmf = MemoryMappedFile::open_read_only(&vector_file).unwrap();
            let repaired_header: VectorStorageHeader = mmf.read_at(0).unwrap();
            assert_eq!(repaired_header.vector_data_offset, reasonable_offset);
        }

        // Check that notes contain expected messages
        assert!(!notes.is_empty());
        assert!(notes
            .iter()
            .any(|note| note.contains("Attempting to repair")));
        assert!(notes
            .iter()
            .any(|note| note.contains("Successfully updated")));
        assert!(notes
            .iter()
            .any(|note| note.contains("verification successful")));
    }

    #[tokio::test]
    async fn test_repair_vector_data_offset_validation_failures() {
        use crate::vector_storage::VectorStorage;
        use tempfile::TempDir;

        let temp_dir = TempDir::new().unwrap();
        let vector_file = temp_dir.path().join("test_vector.dat");

        // Create a vector storage file
        let storage = VectorStorage::create(&vector_file, 4, 100).unwrap();
        drop(storage); // Close file

        let config = ShardexConfig::new()
            .directory_path(temp_dir.path().to_path_buf())
            .vector_size(4);
        let checker = IntegrityChecker::new(temp_dir.path().to_path_buf(), config);

        // Test 1: Offset too small (smaller than header size)
        {
            let mut notes = Vec::new();
            let result = checker
                .repair_vector_data_offset(&vector_file, 50, &mut notes)
                .await;
            assert!(result.is_ok());
            assert!(!result.unwrap()); // Should fail
            assert!(notes.iter().any(|note| note.contains("too small")));
        }

        // Test 2: Offset too large (exceeds file size)
        {
            let file_size = std::fs::metadata(&vector_file).unwrap().len();
            let mut notes = Vec::new();
            let result = checker
                .repair_vector_data_offset(&vector_file, file_size + 100, &mut notes)
                .await;
            assert!(result.is_ok());
            assert!(!result.unwrap()); // Should fail
            assert!(notes.iter().any(|note| note.contains("exceeds file size")));
        }

        // Test 3: Unaligned offset (not 8-byte aligned)
        {
            let mut notes = Vec::new();
            let misaligned_offset = 1000 + 7; // Not divisible by 8
            let result = checker
                .repair_vector_data_offset(&vector_file, misaligned_offset, &mut notes)
                .await;
            assert!(result.is_ok());
            assert!(!result.unwrap()); // Should fail
            assert!(notes.iter().any(|note| note.contains("not 8-byte aligned")));
        }
    }

    #[tokio::test]
    async fn test_repair_vector_data_offset_file_access_failures() {
        use tempfile::TempDir;

        let temp_dir = TempDir::new().unwrap();
        let nonexistent_file = temp_dir.path().join("does_not_exist.dat");

        let config = ShardexConfig::new()
            .directory_path(temp_dir.path().to_path_buf())
            .vector_size(4);
        let checker = IntegrityChecker::new(temp_dir.path().to_path_buf(), config);

        let mut notes = Vec::new();
        let result = checker
            .repair_vector_data_offset(&nonexistent_file, 1000, &mut notes)
            .await;

        // Should handle file access failure gracefully
        assert!(result.is_ok());
        assert!(!result.unwrap()); // Should return false for failure
        assert!(notes
            .iter()
            .any(|note| note.contains("Failed to open file")));
    }
}