oxirs_shacl/validation/

ml_integration.rs

use crate::report::ValidationReport;
use oxirs_core::model::Term;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::time::{Duration, SystemTime};

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TrainingDataPoint {
    pub shape_uri: String,
    pub constraint_type: String,
    pub node_count: usize,
    pub execution_time: Duration,
    pub memory_usage: usize,
    pub validation_result: bool,
    pub complexity_score: f64,
    pub timestamp: SystemTime,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ValidationPrediction {
    pub shape_uri: String,
    pub predicted_violations: usize,
    pub confidence_score: f64,
    pub estimated_duration: Duration,
    pub recommended_strategy: ValidationStrategy,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum ValidationStrategy {
    Parallel,
    Sequential,
    Cached,
    Optimized { batch_size: usize },
    Skip { reason: String },
}

#[derive(Debug, Clone)]
pub struct MLIntegrationConfig {
    pub training_data_export_enabled: bool,
    pub prediction_enabled: bool,
    pub cache_optimization_enabled: bool,
    pub max_training_samples: usize,
    pub prediction_confidence_threshold: f64,
}

impl Default for MLIntegrationConfig {
    fn default() -> Self {
        Self {
            training_data_export_enabled: false,
            prediction_enabled: false,
            cache_optimization_enabled: false,
            max_training_samples: 10000,
            prediction_confidence_threshold: 0.8,
        }
    }
}

pub struct MLIntegrationHooks {
    config: MLIntegrationConfig,
    training_data: Vec<TrainingDataPoint>,
    prediction_cache: HashMap<String, ValidationPrediction>,
    ai_integration_client: Option<Box<dyn AIIntegrationClient>>,
}

pub trait AIIntegrationClient: Send + Sync {
    fn export_training_data(&self, data: &[TrainingDataPoint]) -> Result<(), MLIntegrationError>;
    fn predict_validation(
        &self,
        context: &ValidationContext,
    ) -> Result<ValidationPrediction, MLIntegrationError>;
    fn suggest_constraints(
        &self,
        data_sample: &[Term],
    ) -> Result<Vec<ConstraintSuggestion>, MLIntegrationError>;
    fn optimize_cache_strategy(
        &self,
        usage_patterns: &CacheUsagePatterns,
    ) -> Result<CacheStrategy, MLIntegrationError>;
}

#[derive(Debug, Clone)]
pub struct ValidationContext {
    pub shape_uri: String,
    pub target_count: usize,
    pub constraint_complexity: f64,
    pub historical_performance: Vec<Duration>,
    pub memory_constraints: Option<usize>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ConstraintSuggestion {
    pub constraint_type: String,
    pub property_path: String,
    pub suggested_values: Vec<String>,
    pub confidence_score: f64,
    pub rationale: String,
}

#[derive(Debug, Clone)]
pub struct CacheUsagePatterns {
    pub cache_hit_rate: f64,
    pub most_accessed_shapes: Vec<String>,
    pub memory_usage_trend: Vec<usize>,
    pub eviction_frequency: HashMap<String, usize>,
}

#[derive(Debug, Clone)]
pub struct CacheStrategy {
    pub max_entries: usize,
    pub eviction_policy: EvictionPolicy,
    pub prefetch_patterns: Vec<String>,
    pub memory_threshold: usize,
}

#[derive(Debug, Clone)]
pub enum EvictionPolicy {
    LRU,
    LFU,
    TTL { duration: Duration },
    Predictive { confidence_threshold: f64 },
}

#[derive(Debug, thiserror::Error)]
pub enum MLIntegrationError {
    #[error("Training data export failed: {0}")]
    ExportError(String),
    #[error("Prediction failed: {0}")]
    PredictionError(String),
    #[error("AI client not configured")]
    ClientNotConfigured,
    #[error("Invalid training data: {0}")]
    InvalidData(String),
    #[error("Configuration error: {0}")]
    ConfigError(String),
}

impl MLIntegrationHooks {
    pub fn new(config: MLIntegrationConfig) -> Self {
        Self {
            config,
            training_data: Vec::new(),
            prediction_cache: HashMap::new(),
            ai_integration_client: None,
        }
    }

    pub fn with_ai_client(mut self, client: Box<dyn AIIntegrationClient>) -> Self {
        self.ai_integration_client = Some(client);
        self
    }

    pub fn record_training_data(
        &mut self,
        data_point: TrainingDataPoint,
    ) -> Result<(), MLIntegrationError> {
        if !self.config.training_data_export_enabled {
            return Ok(());
        }

        if self.training_data.len() >= self.config.max_training_samples {
            self.training_data.remove(0);
        }

        self.training_data.push(data_point);
        Ok(())
    }

    pub fn export_training_data(&self) -> Result<(), MLIntegrationError> {
        if let Some(client) = &self.ai_integration_client {
            client.export_training_data(&self.training_data)
        } else {
            Err(MLIntegrationError::ClientNotConfigured)
        }
    }

    pub fn predict_validation(
        &mut self,
        context: &ValidationContext,
    ) -> Result<Option<ValidationPrediction>, MLIntegrationError> {
        if !self.config.prediction_enabled {
            return Ok(None);
        }

        if let Some(cached) = self.prediction_cache.get(&context.shape_uri) {
            return Ok(Some(cached.clone()));
        }

        if let Some(client) = &self.ai_integration_client {
            let prediction = client.predict_validation(context)?;

            if prediction.confidence_score >= self.config.prediction_confidence_threshold {
                self.prediction_cache
                    .insert(context.shape_uri.clone(), prediction.clone());
                Ok(Some(prediction))
            } else {
                Ok(None)
            }
        } else {
            Err(MLIntegrationError::ClientNotConfigured)
        }
    }

    pub fn suggest_constraints(
        &self,
        data_sample: &[Term],
    ) -> Result<Vec<ConstraintSuggestion>, MLIntegrationError> {
        if let Some(client) = &self.ai_integration_client {
            client.suggest_constraints(data_sample)
        } else {
            Err(MLIntegrationError::ClientNotConfigured)
        }
    }

    pub fn optimize_cache_strategy(
        &self,
        usage_patterns: &CacheUsagePatterns,
    ) -> Result<CacheStrategy, MLIntegrationError> {
        if !self.config.cache_optimization_enabled {
            return Ok(CacheStrategy {
                max_entries: 1000,
                eviction_policy: EvictionPolicy::LRU,
                prefetch_patterns: Vec::new(),
                memory_threshold: 100 * 1024 * 1024, // 100MB
            });
        }

        if let Some(client) = &self.ai_integration_client {
            client.optimize_cache_strategy(usage_patterns)
        } else {
            Err(MLIntegrationError::ClientNotConfigured)
        }
    }

    pub fn clear_prediction_cache(&mut self) {
        self.prediction_cache.clear();
    }

    pub fn get_training_data_summary(&self) -> TrainingDataSummary {
        TrainingDataSummary {
            total_samples: self.training_data.len(),
            shape_coverage: self
                .training_data
                .iter()
                .map(|d| d.shape_uri.clone())
                .collect::<std::collections::HashSet<_>>()
                .len(),
            avg_execution_time: self.calculate_avg_execution_time(),
            constraint_type_distribution: self.calculate_constraint_distribution(),
        }
    }

    fn calculate_avg_execution_time(&self) -> Duration {
        if self.training_data.is_empty() {
            return Duration::from_secs(0);
        }

        let total: Duration = self.training_data.iter().map(|d| d.execution_time).sum();

        total / self.training_data.len() as u32
    }

    fn calculate_constraint_distribution(&self) -> HashMap<String, usize> {
        let mut distribution = HashMap::new();
        for data_point in &self.training_data {
            *distribution
                .entry(data_point.constraint_type.clone())
                .or_insert(0) += 1;
        }
        distribution
    }
}

#[derive(Debug, Clone)]
pub struct TrainingDataSummary {
    pub total_samples: usize,
    pub shape_coverage: usize,
    pub avg_execution_time: Duration,
    pub constraint_type_distribution: HashMap<String, usize>,
}

#[derive(Debug, Clone)]
pub struct ValidationPatternAnalysis {
    pub total_patterns: usize,
    pub performance_insights: Vec<PerformanceInsight>,
    pub constraint_recommendations: Vec<ConstraintRecommendation>,
    pub optimization_opportunities: Vec<OptimizationOpportunity>,
}

#[derive(Debug, Clone)]
pub struct PerformanceInsight {
    pub pattern_id: String,
    pub issue_type: String,
    pub description: String,
    pub suggested_action: String,
}

#[derive(Debug, Clone)]
pub struct ConstraintRecommendation {
    pub constraint_type: String,
    pub confidence_score: f64,
    pub rationale: String,
}

#[derive(Debug, Clone)]
pub struct OptimizationOpportunity {
    pub pattern_id: String,
    pub opportunity_type: String,
    pub potential_benefit: String,
    pub implementation_effort: String,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SmartConstraintRecommendation {
    pub constraint_type: String,
    pub target_class: Option<String>,
    pub confidence_score: f64,
    pub rationale: String,
    pub sample_sparql: String,
}

#[derive(Debug, Clone)]
pub struct ValuePatternAnalysis {
    pub avg_string_length: f64,
    pub literal_percentage: f64,
    pub numeric_range: Option<(f64, f64)>,
}

#[derive(Debug, Clone)]
pub struct IntelligentCacheManager {
    cache_usage_patterns: CacheUsagePatterns,
    predictive_cache: HashMap<String, PredictiveCacheEntry>,
    optimization_history: Vec<CacheOptimizationEvent>,
    config: IntelligentCacheConfig,
}

#[derive(Debug, Clone)]
pub struct IntelligentCacheConfig {
    pub enable_predictive_caching: bool,
    pub cache_hit_threshold: f64,
    pub memory_pressure_threshold: usize,
    pub prediction_window: Duration,
    pub max_prediction_entries: usize,
}

impl Default for IntelligentCacheConfig {
    fn default() -> Self {
        Self {
            enable_predictive_caching: true,
            cache_hit_threshold: 0.7,
            memory_pressure_threshold: 100 * 1024 * 1024, // 100MB
            prediction_window: Duration::from_secs(3600), // 1 hour
            max_prediction_entries: 1000,
        }
    }
}

#[derive(Debug, Clone)]
pub struct PredictiveCacheEntry {
    pub shape_uri: String,
    pub predicted_access_time: SystemTime,
    pub confidence_score: f64,
    pub access_pattern: AccessPattern,
    pub warm_cache_hint: WarmCacheHint,
}

#[derive(Debug, Clone)]
pub enum AccessPattern {
    Frequent {
        interval: Duration,
    },
    Burst {
        burst_size: usize,
        interval: Duration,
    },
    Sporadic {
        average_interval: Duration,
    },
    OneTime,
}

#[derive(Debug, Clone)]
pub struct WarmCacheHint {
    pub constraint_types: Vec<String>,
    pub estimated_memory: usize,
    pub priority: CachePriority,
}

#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
pub enum CachePriority {
    Low,
    Medium,
    High,
    Critical,
}

#[derive(Debug, Clone)]
pub struct CacheOptimizationEvent {
    pub timestamp: SystemTime,
    pub event_type: OptimizationEventType,
    pub before_metrics: CacheMetrics,
    pub after_metrics: CacheMetrics,
    pub improvement_percentage: f64,
}

#[derive(Debug, Clone)]
pub enum OptimizationEventType {
    EvictionPolicyChange,
    SizeAdjustment,
    PredictivePrefetch,
    MemoryPressureResponse,
}

#[derive(Debug, Clone)]
pub struct CacheMetrics {
    pub hit_rate: f64,
    pub memory_usage: usize,
    pub avg_access_time: Duration,
    pub eviction_count: usize,
}

impl IntelligentCacheManager {
    pub fn new(config: IntelligentCacheConfig) -> Self {
        Self {
            cache_usage_patterns: CacheUsagePatterns {
                cache_hit_rate: 0.0,
                most_accessed_shapes: Vec::new(),
                memory_usage_trend: Vec::new(),
                eviction_frequency: HashMap::new(),
            },
            predictive_cache: HashMap::new(),
            optimization_history: Vec::new(),
            config,
        }
    }

    pub fn predict_cache_needs(
        &mut self,
        validation_context: &ValidationContext,
    ) -> Result<Vec<PredictiveCacheEntry>, MLIntegrationError> {
        if !self.config.enable_predictive_caching {
            return Ok(Vec::new());
        }

        let mut predictions = Vec::new();

        // Analyze historical patterns
        let access_pattern = self.analyze_access_pattern(&validation_context.shape_uri);

        match access_pattern {
            AccessPattern::Frequent { interval } => {
                let next_access = SystemTime::now() + interval;
                predictions.push(PredictiveCacheEntry {
                    shape_uri: validation_context.shape_uri.clone(),
                    predicted_access_time: next_access,
                    confidence_score: 0.9,
                    access_pattern: access_pattern.clone(),
                    warm_cache_hint: WarmCacheHint {
                        constraint_types: vec!["all".to_string()],
                        estimated_memory: validation_context
                            .memory_constraints
                            .unwrap_or(1024 * 1024),
                        priority: CachePriority::High,
                    },
                });
            }
            AccessPattern::Burst {
                burst_size,
                interval: _,
            } => {
                for i in 0..burst_size {
                    let next_access = SystemTime::now() + Duration::from_secs(i as u64 * 10);
                    predictions.push(PredictiveCacheEntry {
                        shape_uri: format!("{}_{}", validation_context.shape_uri, i),
                        predicted_access_time: next_access,
                        confidence_score: 0.8,
                        access_pattern: access_pattern.clone(),
                        warm_cache_hint: WarmCacheHint {
                            constraint_types: vec!["burst".to_string()],
                            estimated_memory: validation_context
                                .memory_constraints
                                .unwrap_or(512 * 1024),
                            priority: CachePriority::Medium,
                        },
                    });
                }
            }
            _ => {
                // Less predictable patterns get lower priority
                predictions.push(PredictiveCacheEntry {
                    shape_uri: validation_context.shape_uri.clone(),
                    predicted_access_time: SystemTime::now() + Duration::from_secs(300), // 5 minutes
                    confidence_score: 0.5,
                    access_pattern,
                    warm_cache_hint: WarmCacheHint {
                        constraint_types: vec!["opportunistic".to_string()],
                        estimated_memory: 256 * 1024,
                        priority: CachePriority::Low,
                    },
                });
            }
        }

        // Update predictive cache
        for prediction in &predictions {
            if self.predictive_cache.len() < self.config.max_prediction_entries {
                self.predictive_cache
                    .insert(prediction.shape_uri.clone(), prediction.clone());
            }
        }

        Ok(predictions)
    }

    pub fn optimize_cache_strategy(&mut self) -> Result<CacheStrategy, MLIntegrationError> {
        let current_metrics = self.calculate_current_metrics();

        let mut new_strategy = CacheStrategy {
            max_entries: 1000,
            eviction_policy: EvictionPolicy::LRU,
            prefetch_patterns: Vec::new(),
            memory_threshold: self.config.memory_pressure_threshold,
        };

        // Analyze cache hit rate and adjust strategy
        if self.cache_usage_patterns.cache_hit_rate < self.config.cache_hit_threshold {
            // Low hit rate - increase cache size and enable predictive prefetching
            new_strategy.max_entries = (new_strategy.max_entries as f64 * 1.5) as usize;
            new_strategy.eviction_policy = EvictionPolicy::Predictive {
                confidence_threshold: 0.7,
            };

            // Add prefetch patterns based on most accessed shapes
            new_strategy.prefetch_patterns = self.cache_usage_patterns.most_accessed_shapes.clone();
        }

        // Check memory pressure
        if let Some(latest_memory) = self.cache_usage_patterns.memory_usage_trend.last() {
            if *latest_memory > self.config.memory_pressure_threshold {
                // High memory usage - reduce cache size and use aggressive eviction
                new_strategy.max_entries = (new_strategy.max_entries as f64 * 0.7) as usize;
                new_strategy.eviction_policy = EvictionPolicy::LFU;
            }
        }

        // Record optimization event
        let optimization_event = CacheOptimizationEvent {
            timestamp: SystemTime::now(),
            event_type: OptimizationEventType::SizeAdjustment,
            before_metrics: current_metrics.clone(),
            after_metrics: current_metrics, // Will be updated after strategy is applied
            improvement_percentage: 0.0,    // Will be calculated after strategy is applied
        };

        self.optimization_history.push(optimization_event);

        Ok(new_strategy)
    }

    pub fn record_cache_access(&mut self, shape_uri: &str, hit: bool, memory_usage: usize) {
        // Update cache hit rate
        let total_accesses = self
            .cache_usage_patterns
            .eviction_frequency
            .values()
            .sum::<usize>()
            + 1;
        let hits = if hit { 1 } else { 0 };
        self.cache_usage_patterns.cache_hit_rate = hits as f64 / total_accesses as f64;

        // Update most accessed shapes
        if !self
            .cache_usage_patterns
            .most_accessed_shapes
            .contains(&shape_uri.to_string())
        {
            self.cache_usage_patterns
                .most_accessed_shapes
                .push(shape_uri.to_string());
        }

        // Update memory usage trend
        self.cache_usage_patterns
            .memory_usage_trend
            .push(memory_usage);

        // Keep only recent memory usage data (last 100 entries)
        if self.cache_usage_patterns.memory_usage_trend.len() > 100 {
            self.cache_usage_patterns.memory_usage_trend.remove(0);
        }

        // Update eviction frequency
        *self
            .cache_usage_patterns
            .eviction_frequency
            .entry(shape_uri.to_string())
            .or_insert(0) += 1;
    }

    pub fn get_cache_recommendations(&self) -> Vec<CacheRecommendation> {
        let mut recommendations = Vec::new();

        if self.cache_usage_patterns.cache_hit_rate < 0.5 {
            recommendations.push(CacheRecommendation {
                recommendation_type: "increase_cache_size".to_string(),
                rationale: "Low cache hit rate indicates insufficient cache capacity".to_string(),
                expected_improvement: 30.0,
                implementation_effort: "medium".to_string(),
            });
        }

        if let Some(latest_memory) = self.cache_usage_patterns.memory_usage_trend.last() {
            if *latest_memory > self.config.memory_pressure_threshold {
                recommendations.push(CacheRecommendation {
                    recommendation_type: "optimize_memory_usage".to_string(),
                    rationale: "High memory usage detected".to_string(),
                    expected_improvement: 20.0,
                    implementation_effort: "low".to_string(),
                });
            }
        }

        recommendations
    }

    fn analyze_access_pattern(&self, shape_uri: &str) -> AccessPattern {
        // Simple pattern analysis based on eviction frequency
        let access_count = self
            .cache_usage_patterns
            .eviction_frequency
            .get(shape_uri)
            .unwrap_or(&0);

        match access_count {
            0..=2 => AccessPattern::OneTime,
            3..=10 => AccessPattern::Sporadic {
                average_interval: Duration::from_secs(3600),
            },
            11..=50 => AccessPattern::Frequent {
                interval: Duration::from_secs(300),
            },
            _ => AccessPattern::Burst {
                burst_size: 5,
                interval: Duration::from_secs(60),
            },
        }
    }

    fn calculate_current_metrics(&self) -> CacheMetrics {
        CacheMetrics {
            hit_rate: self.cache_usage_patterns.cache_hit_rate,
            memory_usage: *self
                .cache_usage_patterns
                .memory_usage_trend
                .last()
                .unwrap_or(&0),
            avg_access_time: Duration::from_millis(10), // Simplified
            eviction_count: self.cache_usage_patterns.eviction_frequency.values().sum(),
        }
    }
}

#[derive(Debug, Clone)]
pub struct CacheRecommendation {
    pub recommendation_type: String,
    pub rationale: String,
    pub expected_improvement: f64,
    pub implementation_effort: String,
}

pub struct ValidationPatternLearner {
    ml_hooks: MLIntegrationHooks,
    pattern_cache: HashMap<String, ValidationPattern>,
}

#[derive(Debug, Clone)]
pub struct ValidationPattern {
    pub pattern_id: String,
    pub shape_characteristics: ShapeCharacteristics,
    pub performance_profile: PerformanceProfile,
    pub common_violations: Vec<ViolationPattern>,
    pub optimization_hints: Vec<OptimizationHint>,
}

#[derive(Debug, Clone)]
pub struct ShapeCharacteristics {
    pub constraint_count: usize,
    pub complexity_score: f64,
    pub target_type: String,
    pub property_path_depth: usize,
}

#[derive(Debug, Clone)]
pub struct PerformanceProfile {
    pub avg_execution_time: Duration,
    pub memory_usage: usize,
    pub cache_hit_rate: f64,
    pub parallelization_benefit: f64,
}

#[derive(Debug, Clone)]
pub struct ViolationPattern {
    pub violation_type: String,
    pub frequency: f64,
    pub severity: ViolationSeverity,
    pub common_causes: Vec<String>,
}

#[derive(Debug, Clone)]
pub enum ViolationSeverity {
    Low,
    Medium,
    High,
    Critical,
}

#[derive(Debug, Clone)]
pub struct OptimizationHint {
    pub hint_type: OptimizationHintType,
    pub description: String,
    pub expected_improvement: f64,
    pub implementation_complexity: u8, // 1-10 scale
}

#[derive(Debug, Clone)]
pub enum OptimizationHintType {
    Caching,
    Parallelization,
    ConstraintOrdering,
    DataStructure,
    Algorithm,
}

impl ValidationPatternLearner {
    pub fn new(config: MLIntegrationConfig) -> Self {
        Self {
            ml_hooks: MLIntegrationHooks::new(config),
            pattern_cache: HashMap::new(),
        }
    }

    pub fn learn_from_validation(
        &mut self,
        shape_uri: &str,
        execution_time: Duration,
        memory_usage: Option<usize>,
        report: &ValidationReport,
    ) -> Result<(), MLIntegrationError> {
        let training_point = TrainingDataPoint {
            shape_uri: shape_uri.to_string(),
            constraint_type: self.extract_constraint_type(report),
            node_count: report.violations.len(),
            execution_time,
            memory_usage: memory_usage.unwrap_or(0),
            validation_result: report.conforms,
            complexity_score: self.calculate_complexity_score(report),
            timestamp: SystemTime::now(),
        };

        self.ml_hooks.record_training_data(training_point)?;
        self.update_validation_pattern(shape_uri, execution_time, memory_usage, report);
        Ok(())
    }

    pub fn get_pattern(&self, shape_uri: &str) -> Option<&ValidationPattern> {
        self.pattern_cache.get(shape_uri)
    }

    pub fn export_patterns(&self) -> Result<(), MLIntegrationError> {
        self.ml_hooks.export_training_data()
    }

    pub fn export_training_data_to_file(&self, file_path: &str) -> Result<(), MLIntegrationError> {
        let data = &self.ml_hooks.training_data;
        let json_data = serde_json::to_string_pretty(data).map_err(|e| {
            MLIntegrationError::ExportError(format!("JSON serialization failed: {e}"))
        })?;

        std::fs::write(file_path, json_data)
            .map_err(|e| MLIntegrationError::ExportError(format!("File write failed: {e}")))?;

        Ok(())
    }

    pub fn import_training_data_from_file(
        &mut self,
        file_path: &str,
    ) -> Result<(), MLIntegrationError> {
        let content = std::fs::read_to_string(file_path)
            .map_err(|e| MLIntegrationError::ExportError(format!("File read failed: {e}")))?;

        let imported_data: Vec<TrainingDataPoint> = serde_json::from_str(&content)
            .map_err(|e| MLIntegrationError::InvalidData(format!("JSON parsing failed: {e}")))?;

        for data_point in imported_data {
            self.ml_hooks.record_training_data(data_point)?;
        }

        Ok(())
    }

    pub fn analyze_validation_patterns(&self) -> ValidationPatternAnalysis {
        let mut analysis = ValidationPatternAnalysis {
            total_patterns: self.pattern_cache.len(),
            performance_insights: Vec::new(),
            constraint_recommendations: Vec::new(),
            optimization_opportunities: Vec::new(),
        };

        for pattern in self.pattern_cache.values() {
            // Performance insights
            if pattern.performance_profile.avg_execution_time > Duration::from_millis(1000) {
                analysis.performance_insights.push(PerformanceInsight {
                    pattern_id: pattern.pattern_id.clone(),
                    issue_type: "slow_execution".to_string(),
                    description: "Validation taking longer than 1 second".to_string(),
                    suggested_action: "Consider enabling parallel validation or constraint reordering".to_string(),
                });
            }

            if pattern.performance_profile.memory_usage > 10 * 1024 * 1024 {
                // 10MB
                analysis.performance_insights.push(PerformanceInsight {
                    pattern_id: pattern.pattern_id.clone(),
                    issue_type: "high_memory".to_string(),
                    description: "High memory usage detected".to_string(),
                    suggested_action:
                        "Enable streaming validation or implement memory optimization".to_string(),
                });
            }

            // Optimization opportunities
            if pattern.performance_profile.cache_hit_rate < 0.5 {
                analysis
                    .optimization_opportunities
                    .push(OptimizationOpportunity {
                        pattern_id: pattern.pattern_id.clone(),
                        opportunity_type: "caching".to_string(),
                        potential_benefit: "50% performance improvement".to_string(),
                        implementation_effort: "low".to_string(),
                    });
            }
        }

        analysis
    }

    pub fn generate_constraint_recommendations(
        &self,
        data_sample: &[Term],
    ) -> Result<Vec<SmartConstraintRecommendation>, MLIntegrationError> {
        let mut recommendations = Vec::new();

        // Analyze data patterns
        let type_distribution = self.analyze_type_distribution(data_sample);
        let _value_patterns = self.analyze_value_patterns(data_sample);

        // Generate type-based recommendations
        for (rdf_type, count) in type_distribution {
            if count > data_sample.len() / 2 {
                recommendations.push(SmartConstraintRecommendation {
                    constraint_type: "sh:class".to_string(),
                    target_class: Some(rdf_type.clone()),
                    confidence_score: 0.9,
                    rationale: "Majority of nodes are of this type".to_string(),
                    sample_sparql: format!("[] sh:class <{rdf_type}> ."),
                });
            }
        }

        // Generate cardinality recommendations
        if !data_sample.is_empty() {
            recommendations.push(SmartConstraintRecommendation {
                constraint_type: "sh:minCount".to_string(),
                target_class: None,
                confidence_score: 0.8,
                rationale: "Ensure required properties are present".to_string(),
                sample_sparql: "[] sh:minCount 1 .".to_string(),
            });
        }

        Ok(recommendations)
    }

    fn analyze_type_distribution(&self, data_sample: &[Term]) -> HashMap<String, usize> {
        let mut distribution = HashMap::new();

        for term in data_sample {
            if let Term::NamedNode(node) = term {
                let type_name = node.as_str().to_string();
                *distribution.entry(type_name).or_insert(0) += 1;
            }
        }

        distribution
    }

    fn analyze_value_patterns(&self, data_sample: &[Term]) -> ValuePatternAnalysis {
        let mut string_lengths = Vec::new();
        let mut numeric_values = Vec::new();
        let mut literal_count = 0;

        for term in data_sample {
            if let Term::Literal(literal) = term {
                literal_count += 1;
                let value = literal.value();
                string_lengths.push(value.len());

                if let Ok(num) = value.parse::<f64>() {
                    numeric_values.push(num);
                }
            }
        }

        ValuePatternAnalysis {
            avg_string_length: if string_lengths.is_empty() {
                0.0
            } else {
                string_lengths.iter().sum::<usize>() as f64 / string_lengths.len() as f64
            },
            literal_percentage: literal_count as f64 / data_sample.len() as f64,
            numeric_range: if numeric_values.is_empty() {
                None
            } else {
                Some((
                    numeric_values.iter().fold(f64::INFINITY, |a, &b| a.min(b)),
                    numeric_values
                        .iter()
                        .fold(f64::NEG_INFINITY, |a, &b| a.max(b)),
                ))
            },
        }
    }

    fn extract_constraint_type(&self, report: &ValidationReport) -> String {
        // Extract primary constraint type from validation report
        if !report.violations.is_empty() {
            format!("violation_{}", report.violations.len())
        } else {
            "success".to_string()
        }
    }

    fn calculate_complexity_score(&self, report: &ValidationReport) -> f64 {
        // Simple complexity scoring based on report structure
        let base_score = 1.0;
        let violation_factor = report.violations.len() as f64 * 0.1;
        base_score + violation_factor
    }

    fn update_validation_pattern(
        &mut self,
        shape_uri: &str,
        execution_time: Duration,
        memory_usage: Option<usize>,
        report: &ValidationReport,
    ) {
        let pattern = ValidationPattern {
            pattern_id: format!("pattern_{shape_uri}"),
            shape_characteristics: ShapeCharacteristics {
                constraint_count: 1, // Simplified
                complexity_score: self.calculate_complexity_score(report),
                target_type: "unknown".to_string(),
                property_path_depth: 1,
            },
            performance_profile: PerformanceProfile {
                avg_execution_time: execution_time,
                memory_usage: memory_usage.unwrap_or(0),
                cache_hit_rate: 0.0, // Would be calculated from actual cache metrics
                parallelization_benefit: 0.0,
            },
            common_violations: Vec::new(),
            optimization_hints: Vec::new(),
        };

        self.pattern_cache.insert(shape_uri.to_string(), pattern);
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_ml_integration_config_default() {
        let config = MLIntegrationConfig::default();
        assert!(!config.training_data_export_enabled);
        assert!(!config.prediction_enabled);
        assert_eq!(config.max_training_samples, 10000);
        assert_eq!(config.prediction_confidence_threshold, 0.8);
    }

    #[test]
    fn test_ml_integration_hooks_creation() {
        let config = MLIntegrationConfig::default();
        let hooks = MLIntegrationHooks::new(config);
        assert_eq!(hooks.training_data.len(), 0);
        assert_eq!(hooks.prediction_cache.len(), 0);
    }

    #[test]
    fn test_training_data_recording() {
        let config = MLIntegrationConfig {
            training_data_export_enabled: true,
            ..Default::default()
        };

        let mut hooks = MLIntegrationHooks::new(config);

        let data_point = TrainingDataPoint {
            shape_uri: "http://example.org/shape".to_string(),
            constraint_type: "sh:minCount".to_string(),
            node_count: 100,
            execution_time: Duration::from_millis(50),
            memory_usage: 1024,
            validation_result: true,
            complexity_score: 2.5,
            timestamp: SystemTime::now(),
        };

        assert!(hooks.record_training_data(data_point).is_ok());
        assert_eq!(hooks.training_data.len(), 1);
    }

    #[test]
    fn test_validation_pattern_learner_creation() {
        let config = MLIntegrationConfig::default();
        let learner = ValidationPatternLearner::new(config);
        assert_eq!(learner.pattern_cache.len(), 0);
    }

    #[test]
    fn test_training_data_summary() {
        let config = MLIntegrationConfig {
            training_data_export_enabled: true,
            ..Default::default()
        };

        let mut hooks = MLIntegrationHooks::new(config);

        let data_point = TrainingDataPoint {
            shape_uri: "http://example.org/shape".to_string(),
            constraint_type: "sh:minCount".to_string(),
            node_count: 100,
            execution_time: Duration::from_millis(50),
            memory_usage: 1024,
            validation_result: true,
            complexity_score: 2.5,
            timestamp: SystemTime::now(),
        };

        hooks
            .record_training_data(data_point)
            .expect("training should succeed");

        let summary = hooks.get_training_data_summary();
        assert_eq!(summary.total_samples, 1);
        assert_eq!(summary.shape_coverage, 1);
        assert_eq!(summary.avg_execution_time, Duration::from_millis(50));
    }

    #[test]
    fn test_intelligent_cache_manager_creation() {
        let config = IntelligentCacheConfig::default();
        let cache_manager = IntelligentCacheManager::new(config);
        assert!(cache_manager.config.enable_predictive_caching);
        assert_eq!(cache_manager.config.cache_hit_threshold, 0.7);
        assert_eq!(cache_manager.predictive_cache.len(), 0);
    }

    #[test]
    fn test_cache_access_recording() {
        let config = IntelligentCacheConfig::default();
        let mut cache_manager = IntelligentCacheManager::new(config);

        cache_manager.record_cache_access("test_shape", true, 1024);
        assert!(cache_manager
            .cache_usage_patterns
            .most_accessed_shapes
            .contains(&"test_shape".to_string()));
        assert_eq!(
            cache_manager.cache_usage_patterns.memory_usage_trend.len(),
            1
        );
    }

    #[test]
    fn test_predictive_caching() {
        let config = IntelligentCacheConfig::default();
        let mut cache_manager = IntelligentCacheManager::new(config);

        let validation_context = ValidationContext {
            shape_uri: "test_shape".to_string(),
            target_count: 100,
            constraint_complexity: 2.5,
            historical_performance: vec![Duration::from_millis(100)],
            memory_constraints: Some(1024 * 1024),
        };

        let predictions = cache_manager
            .predict_cache_needs(&validation_context)
            .expect("validation should succeed");
        assert!(!predictions.is_empty());
        assert_eq!(predictions[0].shape_uri, "test_shape");
    }

    #[test]
    fn test_cache_optimization() {
        let config = IntelligentCacheConfig::default();
        let mut cache_manager = IntelligentCacheManager::new(config);

        // Simulate low cache hit rate
        cache_manager.cache_usage_patterns.cache_hit_rate = 0.3;

        let strategy = cache_manager
            .optimize_cache_strategy()
            .expect("optimization should succeed");
        assert!(strategy.max_entries > 1000); // Should increase cache size
        assert_eq!(cache_manager.optimization_history.len(), 1);
    }

    #[test]
    fn test_cache_recommendations() {
        let config = IntelligentCacheConfig::default();
        let mut cache_manager = IntelligentCacheManager::new(config);

        // Simulate low cache hit rate
        cache_manager.cache_usage_patterns.cache_hit_rate = 0.3;

        let recommendations = cache_manager.get_cache_recommendations();
        assert!(!recommendations.is_empty());
        assert_eq!(
            recommendations[0].recommendation_type,
            "increase_cache_size"
        );
    }

    #[test]
    fn test_access_pattern_analysis() {
        let config = IntelligentCacheConfig::default();
        let mut cache_manager = IntelligentCacheManager::new(config);

        // Simulate frequent access
        for _ in 0..20 {
            cache_manager.record_cache_access("frequent_shape", true, 1024);
        }

        let pattern = cache_manager.analyze_access_pattern("frequent_shape");
        match pattern {
            AccessPattern::Frequent { .. } => (), // Expected frequent access pattern
            _ => panic!("Expected frequent access pattern"),
        }
    }
}