reasonkit-core 0.1.8

//! Multi-Model Semantic Validation
//!
//! Cross-model validation for consistency and correctness of LLM outputs.
//! Validates outputs across multiple LLM providers (Claude, GPT, Gemini, etc.)
//! to detect hallucinations, inconsistencies, and model-specific biases.
//!
//! ## Architecture
//!
//! ```text
//! Multi-Model Semantic Validation Stack
//! ┌─────────────────────────────────────────────────────────────┐
//! │ Cross-Model Consistency Check                             │
//! │ • Semantic similarity across providers                      │
//! │ • Outlier detection                                         │
//! │ • Consensus voting                                          │
//! └─────────────────────────────────────────────────────────────┘
//! ┌─────────────────────────────────────────────────────────────┐
//! │ Adversarial Prompt Generation                              │
//! │ • Contradiction injection                                  │
//! │ • Logical fallacy embedding                                │
//! │ • Ambiguity amplification                                  │
//! └─────────────────────────────────────────────────────────────┘
//! ┌─────────────────────────────────────────────────────────────┐
//! │ Semantic Equivalence Validation                             │
//! │ • Meaning-preserving transformations                       │
//! │ • Paraphrase detection                                     │
//! │ • Information retention                                    │
//! └─────────────────────────────────────────────────────────────┘
//! ```
//!
//! ## Usage
//!
//! ```rust,ignore
//! use reasonkit::semantic_validation::MultiModelValidator;
//! use reasonkit::semantic_validation::AdversarialPromptGenerator;
//!
//! // Create validator with Claude, GPT, Gemini
//! let validator = MultiModelValidator::new(vec![
//!     LlmProvider::Anthropic,
//!     LlmProvider::OpenAI,
//!     LlmProvider::GoogleGemini,
//! ]);
//!
//! // Validate across models
//! let result = validator.validate_consistency(
//!     "What is the capital of France?",
//!     Some("Paris"),
//! ).await?;
//!
//! if result.consensus_reached {
//!     println!("✅ All models agree: {}", result.consensus_answer);
//! } else {
//!     println!("⚠️  Model disagreement detected");
//!     for (model, answer) in result.responses {
//!         println!("  {}: {}", model, answer);
//!     }
//! }
//!
//! // Generate adversarial prompts
//! let generator = AdversarialPromptGenerator::new();
//! let adversarial_prompts = generator.generate_test_suite(
//!     "Explain quantum computing",
//!     vec![AttackStrategy::ContradictionInjection],
//! ).await?;
//! ```

mod cross_model_validator;
mod adversarial_generator;
mod semantic_equivalence;
mod consistency_metrics;

pub use cross_model_validator::*;
pub use adversarial_generator::*;
pub use semantic_equivalence::*;
pub use consistency_metrics::*;

use crate::error::Result;
use serde::{Deserialize, Serialize};

/// Multi-model validation configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SemanticValidationConfig {
    /// Minimum number of models required for consensus (≥ 3)
    pub min_models: usize,
    /// Semantic similarity threshold (0.0-1.0)
    pub similarity_threshold: f64,
    /// Maximum allowed semantic variance
    pub max_variance: f64,
    /// Whether to enable adversarial prompt generation
    pub enable_adversarial_generation: bool,
    /// Adversarial generation intensity (0.0-1.0)
    pub adversarial_intensity: f64,
    /// Whether to cache validation results
    pub enable_caching: bool,
    /// Cache TTL in seconds
    pub cache_ttl_secs: u64,
}

impl Default for SemanticValidationConfig {
    fn default() -> Self {
        Self {
            min_models: 3,
            similarity_threshold: 0.85,
            max_variance: 0.15,
            enable_adversarial_generation: true,
            adversarial_intensity: 0.3,
            enable_caching: true,
            cache_ttl_secs: 3600, // 1 hour
        }
    }
}

/// Validation result for a single model
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ModelValidation {
    /// Model identifier
    pub model_id: String,
    /// Provider name
    pub provider: String,
    /// Response validation
    pub response: String,
    /// Confidence score (0.0-1.0)
    pub confidence: f64,
    /// Factual accuracy score (if available)
    pub factual_accuracy: Option<f64>,
    /// Logical consistency score (if available)
    pub logical_consistency: Option<f64>,
    /// Response latency in milliseconds
    pub latency_ms: u64,
    /// Any warnings or issues
    pub warnings: Vec<String>,
}

/// Multi-model validation result
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MultiModelValidation {
    /// Original query
    pub query: String,
    /// Target answer (if provided)
    pub target_answer: Option<String>,
    /// Individual model validations
    pub model_validations: Vec<ModelValidation>,
    /// Consensus metrics
    pub consensus: ConsensusMetrics,
    /// Semantic variance across models
    pub semantic_variance: f64,
    /// Outlier detection results
    pub outliers: Vec<OutlierDetection>,
    /// Time to consensus in milliseconds
    pub time_to_consensus_ms: u64,
    /// Recommendations for improvement
    pub recommendations: Vec<ValidationRecommendation>,
}

/// Consensus metrics across models
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ConsensusMetrics {
    /// Whether consensus was reached
    pub reached: bool,
    /// Consensus score (0.0-1.0)
    pub score: f64,
    /// Number of agreeing models
    pub agreeing_models: usize,
    /// Number of disagreeing models
    pub disagreeing_models: usize,
    /// Most common response (if consensus reached)
    pub consensus_response: Option<String>,
    /// Consistency of that response (0.0-1.0)
    pub consensus_consistency: f64,
}

/// Outlier detection result
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct OutlierDetection {
    /// Model identified as outlier
    pub outlier_model: String,
    /// Distance from consensus
    pub distance_from_consensus: f64,
    /// Reason for outlier classification
    pub reason: OutlierReason,
    /// Suggested correction (if applicable)
    pub suggested_correction: Option<String>,
}

/// Outlier classification reason
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum OutlierReason {
    SemanticDistance,
    FactualInaccuracy,
    LogicalInconsistency,
    ResponseAnomaly,
    AdversarialSusceptibility,
}

/// Validation recommendation
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ValidationRecommendation {
    /// Priority level
    pub priority: RecommendationPriority,
    /// Recommendation text
    pub text: String,
    /// Suggested action
    pub action: String,
    /// Expected improvement (0.0-1.0)
    pub expected_improvement: f64,
}

/// Recommendation priority
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum RecommendationPriority {
    Critical,
    High,
    Medium,
    Low,
}

/// Adversarial prompt generation strategies
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum AttackStrategy {
    /// Inject logical contradictions
    ContradictionInjection,
    /// Embed logical fallacies
    LogicalFallacyEmbedding,
    /// Increase ambiguity
    AmbiguityAmplification,
    /// Boundary value exploration
    BoundaryExploration,
    /// Semantic pollution
    SemanticPollution,
    /// Format manipulation
    FormatManipulation,
}

/// Adversarial test suite
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AdversarialTestSuite {
    /// Base prompt
    pub base_prompt: String,
    /// Generated adversarial variants
    pub adversarial_variants: Vec<AdversarialVariant>,
    /// Attack strategies used
    pub strategies: Vec<AttackStrategy>,
    /// Success rates per variant
    pub success_rates: Vec<f64>,
    /// Model susceptibility scores
    pub susceptibility_scores: Vec<ModelSusceptibility>,
}

/// Adversarial variant
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AdversarialVariant {
    /// Variant identifier
    pub id: String,
    /// Adversarial prompt
    pub prompt: String,
    /// Strategy used
    pub strategy: AttackStrategy,
    /// Intensity level (0.0-1.0)
    pub intensity: f64,
    /// Expected impact description
    pub expected_impact: String,
}

/// Model susceptibility to adversarial attacks
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ModelSusceptibility {
    /// Model identifier
    pub model_id: String,
    /// Overall susceptibility score (0.0-1.0, higher = more susceptible)
    pub score: f64,
    /// Per-strategy susceptibility
    pub strategy_scores: Vec<StrategySusceptibility>,
    /// Vulnerability patterns detected
    pub vulnerability_patterns: Vec<VulnerabilityPattern>,
}

/// Strategy-specific susceptibility
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct StrategySusceptibility {
    /// Attack strategy
    pub strategy: AttackStrategy,
    /// Susceptibility score for this strategy
    pub score: f64,
    /// Attack success rate
    pub success_rate: f64,
}

/// Vulnerability pattern
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct VulnerabilityPattern {
    /// Pattern identifier
    pub pattern_id: String,
    /// Pattern description
    pub description: String,
    /// Severity level
    pub severity: VulnerabilitySeverity,
    /// Mitigation suggestions
    pub mitigations: Vec<String>,
}

/// Vulnerability severity
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum VulnerabilitySeverity {
    Critical,
    High,
    Medium,
    Low,
}

/// Core validation trait
pub trait SemanticValidator {
    /// Validate a query across multiple models
    fn validate_query(
        &self,
        query: &str,
        target_answer: Option<&str>,
    ) -> Result<MultiModelValidation>;
    
    /// Check for consistency with previous validations
    fn check_consistency(
        &self,
        current: &MultiModelValidation,
        previous: &MultiModelValidation,
    ) -> ConsistencyResult;
    
    /// Generate adversarial test suite
    fn generate_adversarial_suite(
        &self,
        base_prompt: &str,
        strategies: Vec<AttackStrategy>,
    ) -> Result<AdversarialTestSuite>;
}

/// Consistency check result
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ConsistencyResult {
    /// Whether consistency is maintained
    pub consistent: bool,
    /// Consistency score (0.0-1.0)
    pub score: f64,
    /// Drift amount detected
    pub drift_amount: f64,
    /// Drift direction
    pub drift_direction: DriftDirection,
    /// Recommendations for correction
    pub corrections: Vec<ConsistencyCorrection>,
}

/// Drift direction
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum DriftDirection {
    Improvement,
    Regression,
    SemanticShift,
    NoSignificantDrift,
}

/// Consistency correction
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ConsistencyCorrection {
    /// Correction type
    pub correction_type: CorrectionType,
    /// Suggested action
    pub suggested_action: String,
    /// Expected impact
    pub expected_impact: f64,
}

/// Correction type
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum CorrectionType {
    SemanticAlignment,
    ModelRetraining,
    PromptEnhancement,
    ValidationStrengthening,
}

/// Semantic validation utilities
pub mod utils {
    use super::*;
    
    /// Calculate semantic similarity between two texts
    /// Returns score between 0.0 (completely different) and 1.0 (identical meaning)
    pub fn calculate_semantic_similarity(text1: &str, text2: &str) -> f64 {
        // In production, this would use embedding models
        // Simplified implementation for demonstration
        let words1: Vec<&str> = text1.split_whitespace().collect();
        let words2: Vec<&str> = text2.split_whitespace().collect();
        
        let common_words: Vec<&str> = words1
            .iter()
            .filter(|w| words2.contains(w))
            .copied()
            .collect();
        
        let total_words = words1.len().max(words2.len());
        
        if total_words == 0 {
            return 1.0; // Both empty considers identical
        }
        
        common_words.len() as f64 / total_words as f64
    }
    
    /// Detect semantic drift between validation results
    pub fn detect_semantic_drift(
        old: &MultiModelValidation,
        new: &MultiModelValidation,
    ) -> DriftAnalysis {
        let similarity = calculate_semantic_similarity(
            &old.query,
            &new.query,
        );
        
        let consensus_diff = (old.consensus.score - new.consensus.score).abs();
        let variance_diff = (old.semantic_variance - new.semantic_variance).abs();
        
        DriftAnalysis {
            semantic_similarity: similarity,
            consensus_change: consensus_diff,
            variance_change: variance_diff,
            significant_drift: consensus_diff > 0.1 || variance_diff > 0.1,
        }
    }
    
    /// Drift analysis
    #[derive(Debug, Clone)]
    pub struct DriftAnalysis {
        pub semantic_similarity: f64,
        pub consensus_change: f64,
        pub variance_change: f64,
        pub significant_drift: bool,
    }
}