codex_memory/memory/

cognitive_memory_system.rs

//! Cognitive Memory System Integration
//!
//! This module provides a unified interface for the SOTA cognitive memory system,
//! integrating all cognitive enhancement components into a cohesive architecture.
//!
//! ## System Architecture
//!
//! The Cognitive Memory System combines several research-backed components:
//!
//! ### Core Components
//! 1. **Three-Component Scoring**: Real-time memory importance calculation
//! 2. **Cognitive Consolidation**: Enhanced memory strengthening with spacing effects
//! 3. **Reflection Engine**: Meta-memory and insight generation
//! 4. **Knowledge Graph**: Semantic relationship tracking
//! 5. **Loop Prevention**: Quality control and duplicate detection
//!
//! ### Research Foundation
//! - **Park et al. (2023)**: Generative agents three-component scoring
//! - **Ebbinghaus (1885)**: Forgetting curve and spaced repetition
//! - **Collins & Loftus (1975)**: Semantic network theory
//! - **Flavell (1979)**: Metacognition and reflection processes
//! - **Bjork (1994)**: Desirable difficulties and testing effects
//!
//! ## Usage Example
//!
//! ```rust
//! use codex::memory::CognitiveMemorySystem;
//!
//! let mut system = CognitiveMemorySystem::new(repository).await?;
//!
//! // Store memory with cognitive enhancement
//! let memory = system.store_memory_with_cognitive_processing(
//!     "Important information to remember",
//!     context
//! ).await?;
//!
//! // Retrieve with enhanced scoring
//! let results = system.cognitive_search(
//!     "find related information",
//!     search_context
//! ).await?;
//!
//! // Trigger reflection for insight generation
//! let insights = system.trigger_reflection_if_needed().await?;
//! ```

use super::background_reflection_service::{
    BackgroundReflectionConfig, BackgroundReflectionService, ReflectionServiceMetrics,
};
use super::cognitive_consolidation::{
    CognitiveConsolidationConfig, CognitiveConsolidationEngine, CognitiveConsolidationResult,
    RetrievalContext,
};
use super::error::Result;
use super::insight_loop_prevention::{
    LoopPreventionConfig, LoopPreventionEngine, PreventionAction,
};
use super::models::*;
use super::reflection_engine::{Insight, ReflectionConfig, ReflectionEngine, ReflectionSession};
use super::repository::MemoryRepository;
use super::three_component_scoring::{
    EnhancedSearchService, ScoringContext, ThreeComponentConfig, ThreeComponentEngine,
};

use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use std::sync::Arc;
use tokio::sync::RwLock;
use tracing::{debug, info, warn};
use uuid::Uuid;

/// Configuration for the complete cognitive memory system
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CognitiveMemoryConfig {
    /// Three-component scoring configuration
    pub scoring_config: ThreeComponentConfig,

    /// Cognitive consolidation configuration
    pub consolidation_config: CognitiveConsolidationConfig,

    /// Reflection engine configuration
    pub reflection_config: ReflectionConfig,

    /// Loop prevention configuration
    pub loop_prevention_config: LoopPreventionConfig,

    /// Background reflection service configuration
    pub background_reflection_config: BackgroundReflectionConfig,

    /// Enable automatic cognitive processing
    pub enable_auto_processing: bool,

    /// Enable background reflection monitoring
    pub enable_background_reflection: bool,

    /// Minimum time between automatic processing (minutes)
    pub auto_processing_interval_minutes: u64,

    /// Performance monitoring configuration
    pub enable_performance_monitoring: bool,

    /// Maximum concurrent cognitive operations
    pub max_concurrent_operations: usize,
}

impl Default for CognitiveMemoryConfig {
    fn default() -> Self {
        Self {
            scoring_config: ThreeComponentConfig::default(),
            consolidation_config: CognitiveConsolidationConfig::default(),
            reflection_config: ReflectionConfig::default(),
            loop_prevention_config: LoopPreventionConfig::default(),
            background_reflection_config: BackgroundReflectionConfig::default(),
            enable_auto_processing: true,
            enable_background_reflection: true,
            auto_processing_interval_minutes: 30,
            enable_performance_monitoring: true,
            max_concurrent_operations: 10,
        }
    }
}

/// Performance metrics for the cognitive system
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CognitivePerformanceMetrics {
    pub total_memories_processed: u64,
    pub total_insights_generated: u64,
    pub total_reflections_completed: u64,
    pub average_scoring_time_ms: f64,
    pub average_consolidation_time_ms: f64,
    pub average_reflection_time_ms: f64,
    pub loop_prevention_blocks: u64,
    pub quality_rejections: u64,
    pub system_uptime_hours: f64,
    pub last_updated: DateTime<Utc>,
}

impl Default for CognitivePerformanceMetrics {
    fn default() -> Self {
        Self {
            total_memories_processed: 0,
            total_insights_generated: 0,
            total_reflections_completed: 0,
            average_scoring_time_ms: 0.0,
            average_consolidation_time_ms: 0.0,
            average_reflection_time_ms: 0.0,
            loop_prevention_blocks: 0,
            quality_rejections: 0,
            system_uptime_hours: 0.0,
            last_updated: Utc::now(),
        }
    }
}

/// Enhanced memory storage request with cognitive context
#[derive(Debug, Clone)]
pub struct CognitiveMemoryRequest {
    pub content: String,
    pub embedding: Option<Vec<f32>>,
    pub importance_score: Option<f64>,
    pub metadata: Option<serde_json::Value>,
    pub retrieval_context: RetrievalContext,
    pub enable_immediate_consolidation: bool,
    pub enable_quality_assessment: bool,
}

/// Enhanced memory with cognitive processing results
#[derive(Debug, Clone)]
pub struct CognitiveMemoryResult {
    pub memory: Memory,
    pub consolidation_result: Option<CognitiveConsolidationResult>,
    pub quality_assessment: Option<super::insight_loop_prevention::QualityAssessment>,
    pub processing_time_ms: u64,
    pub cognitive_flags: CognitiveFlags,
}

/// Flags indicating cognitive processing status
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct CognitiveFlags {
    pub consolidation_applied: bool,
    pub reflection_triggered: bool,
    pub quality_validated: bool,
    pub loop_prevention_checked: bool,
    pub three_component_scored: bool,
}

/// Main cognitive memory system orchestrator
pub struct CognitiveMemorySystem {
    repository: Arc<MemoryRepository>,
    config: CognitiveMemoryConfig,

    // Cognitive engines
    scoring_engine: ThreeComponentEngine,
    consolidation_engine: CognitiveConsolidationEngine,
    reflection_engine: Arc<RwLock<ReflectionEngine>>,
    loop_prevention_engine: Arc<RwLock<LoopPreventionEngine>>,
    search_service: EnhancedSearchService,
    background_reflection_service: Arc<BackgroundReflectionService>,

    // System state
    performance_metrics: Arc<RwLock<CognitivePerformanceMetrics>>,
    last_background_processing: Arc<RwLock<DateTime<Utc>>>,
    system_start_time: DateTime<Utc>,
}

impl CognitiveMemorySystem {
    /// Create a new cognitive memory system
    pub async fn new(
        repository: Arc<MemoryRepository>,
        config: CognitiveMemoryConfig,
    ) -> Result<Self> {
        info!("Initializing Cognitive Memory System with enhanced features");

        // Initialize cognitive engines
        let scoring_engine = ThreeComponentEngine::new(config.scoring_config.clone())?;
        let consolidation_engine =
            CognitiveConsolidationEngine::new(config.consolidation_config.clone());
        let reflection_engine = Arc::new(RwLock::new(ReflectionEngine::new(
            config.reflection_config.clone(),
            repository.clone(),
        )));
        let loop_prevention_engine = Arc::new(RwLock::new(LoopPreventionEngine::new(
            config.loop_prevention_config.clone(),
        )));
        let search_service = EnhancedSearchService::new(config.scoring_config.clone())?;

        // Initialize background reflection service
        let background_reflection_service = Arc::new(BackgroundReflectionService::new(
            config.background_reflection_config.clone(),
            repository.clone(),
            config.reflection_config.clone(),
            config.loop_prevention_config.clone(),
        ));

        let performance_metrics = Arc::new(RwLock::new(CognitivePerformanceMetrics::default()));
        let last_background_processing = Arc::new(RwLock::new(Utc::now()));
        let system_start_time = Utc::now();

        info!("Cognitive Memory System initialized successfully");

        let system = Self {
            repository,
            config,
            scoring_engine,
            consolidation_engine,
            reflection_engine,
            loop_prevention_engine,
            search_service,
            background_reflection_service,
            performance_metrics,
            last_background_processing,
            system_start_time,
        };

        // Start background reflection service if enabled
        if system.config.enable_background_reflection {
            system.background_reflection_service.start().await?;
            info!("Background reflection service started");
        }

        Ok(system)
    }

    /// Store memory with full cognitive processing
    pub async fn store_memory_with_cognitive_processing(
        &self,
        request: CognitiveMemoryRequest,
    ) -> Result<CognitiveMemoryResult> {
        let start_time = std::time::Instant::now();
        let mut cognitive_flags = CognitiveFlags::default();

        info!("Processing memory storage with cognitive enhancements");

        // Create base memory
        let create_request = CreateMemoryRequest {
            content: request.content.clone(),
            embedding: request.embedding.clone(),
            tier: Some(MemoryTier::Working),
            importance_score: request.importance_score,
            metadata: request.metadata.clone(),
            parent_id: None,
            expires_at: None,
        };

        let mut memory = self.repository.create_memory(create_request).await?;

        // Apply three-component scoring
        let scoring_context = ScoringContext {
            query_embedding: request.embedding.clone().map(pgvector::Vector::from),
            context_factors: request.retrieval_context.environmental_factors.clone(),
            query_time: Utc::now(),
            user_preferences: std::collections::HashMap::new(),
        };

        if let Ok(scoring_result) =
            self.scoring_engine
                .calculate_score(&memory, &scoring_context, false)
        {
            // Update memory with scores (would be persisted in production)
            cognitive_flags.three_component_scored = true;
            debug!(
                "Three-component scoring applied: combined score {:.3}",
                scoring_result.combined_score
            );
        }

        // Apply cognitive consolidation if enabled
        let consolidation_result = if request.enable_immediate_consolidation {
            let similar_memories = self.find_similar_memories(&memory, 10).await?;

            match self
                .consolidation_engine
                .calculate_cognitive_consolidation(
                    &memory,
                    &request.retrieval_context,
                    &similar_memories,
                )
                .await
            {
                Ok(result) => {
                    self.consolidation_engine
                        .apply_consolidation_results(&mut memory, &result, &self.repository)
                        .await?;
                    cognitive_flags.consolidation_applied = true;
                    debug!(
                        "Cognitive consolidation applied: strength {:.3}",
                        result.new_consolidation_strength
                    );
                    Some(result)
                }
                Err(e) => {
                    warn!("Consolidation failed: {}", e);
                    None
                }
            }
        } else {
            None
        };

        // Quality assessment if enabled
        let quality_assessment = if request.enable_quality_assessment {
            // Create a mock insight for quality assessment
            let mock_insight = Insight {
                id: Uuid::new_v4(),
                insight_type: super::reflection_engine::InsightType::Pattern,
                content: request.content.clone(),
                confidence_score: 0.8,
                source_memory_ids: vec![memory.id],
                related_concepts: Vec::new(), // Would extract from content
                knowledge_graph_nodes: Vec::new(),
                importance_score: memory.importance_score,
                generated_at: Utc::now(),
                validation_metrics: super::reflection_engine::ValidationMetrics {
                    novelty_score: 0.8,
                    coherence_score: 0.9,
                    evidence_strength: 0.7,
                    semantic_richness: 0.6,
                    predictive_power: 0.5,
                },
            };

            match self
                .loop_prevention_engine
                .write()
                .await
                .validate_insight(&mock_insight)
            {
                Ok(loop_result) => {
                    cognitive_flags.quality_validated = true;
                    cognitive_flags.loop_prevention_checked = true;

                    if loop_result.prevention_action == PreventionAction::RejectInsight {
                        warn!("Memory quality assessment failed - would be rejected as insight");
                    }

                    Some(super::insight_loop_prevention::QualityAssessment {
                        novelty_score: 0.8,
                        coherence_score: 0.9,
                        evidence_strength: 0.7,
                        semantic_richness: 0.6,
                        predictive_power: 0.5,
                        overall_quality: 0.74,
                        quality_factors: vec!["High coherence".to_string()],
                        deficiency_reasons: Vec::new(),
                    })
                }
                Err(e) => {
                    warn!("Quality assessment failed: {}", e);
                    None
                }
            }
        } else {
            None
        };

        // Check if reflection should be triggered
        if self.config.enable_auto_processing {
            if let Ok(should_reflect) = self
                .reflection_engine
                .read()
                .await
                .should_trigger_reflection()
                .await
            {
                if should_reflect.is_some() {
                    cognitive_flags.reflection_triggered = true;
                    debug!("Reflection will be triggered in background");

                    // Trigger background reflection (non-blocking)
                    let reflection_engine = self.reflection_engine.clone();
                    let trigger_reason = should_reflect.unwrap();
                    tokio::spawn(async move {
                        let mut engine = reflection_engine.write().await;
                        match engine.execute_reflection(trigger_reason).await {
                            Ok(session) => {
                                info!(
                                    "Background reflection completed: {} insights generated",
                                    session.generated_insights.len()
                                );
                            }
                            Err(e) => {
                                warn!("Background reflection failed: {}", e);
                            }
                        }
                    });
                }
            }
        }

        let processing_time = start_time.elapsed().as_millis() as u64;

        // Update performance metrics
        self.update_performance_metrics(processing_time, &cognitive_flags)
            .await;

        info!(
            "Cognitive memory processing completed in {}ms",
            processing_time
        );

        Ok(CognitiveMemoryResult {
            memory,
            consolidation_result,
            quality_assessment,
            processing_time_ms: processing_time,
            cognitive_flags,
        })
    }

    /// Perform enhanced search with cognitive scoring
    pub async fn cognitive_search(
        &self,
        query: &str,
        context: ScoringContext,
        limit: Option<i32>,
    ) -> Result<Vec<super::three_component_scoring::EnhancedSearchResult>> {
        let start_time = std::time::Instant::now();

        info!("Performing cognitive search for: {}", query);

        // Create base search request
        let search_request = SearchRequest {
            query_text: Some(query.to_string()),
            query_embedding: context
                .query_embedding
                .as_ref()
                .map(|v| v.as_slice().to_vec()),
            search_type: Some(SearchType::Hybrid),
            limit,
            explain_score: Some(true),
            ..Default::default()
        };

        // Perform base search
        let search_response = self.repository.search_memories(search_request).await?;

        // Apply cognitive ranking
        let enhanced_results =
            self.search_service
                .rank_search_results(search_response.results, &context, true)?;

        let processing_time = start_time.elapsed().as_millis() as u64;

        debug!(
            "Cognitive search completed in {}ms, {} results",
            processing_time,
            enhanced_results.len()
        );

        Ok(enhanced_results)
    }

    /// Manually trigger reflection for insight generation
    pub async fn trigger_reflection(&self, reason: String) -> Result<ReflectionSession> {
        info!("Manually triggering reflection: {}", reason);

        let mut reflection_engine = self.reflection_engine.write().await;
        let session = reflection_engine.execute_reflection(reason).await?;

        // Process generated insights through loop prevention
        let mut loop_prevention = self.loop_prevention_engine.write().await;
        for insight in &session.generated_insights {
            match loop_prevention.validate_insight(insight) {
                Ok(validation_result) => {
                    if validation_result.prevention_action == PreventionAction::Allow {
                        // Register valid insight
                        let quality = super::insight_loop_prevention::QualityAssessment {
                            novelty_score: insight.validation_metrics.novelty_score,
                            coherence_score: insight.validation_metrics.coherence_score,
                            evidence_strength: insight.validation_metrics.evidence_strength,
                            semantic_richness: insight.validation_metrics.semantic_richness,
                            predictive_power: insight.validation_metrics.predictive_power,
                            overall_quality: (insight.validation_metrics.novelty_score
                                + insight.validation_metrics.coherence_score
                                + insight.validation_metrics.evidence_strength
                                + insight.validation_metrics.semantic_richness
                                + insight.validation_metrics.predictive_power)
                                / 5.0,
                            quality_factors: vec!["Validated through reflection".to_string()],
                            deficiency_reasons: Vec::new(),
                        };

                        if let Err(e) = loop_prevention.register_insight(insight, quality) {
                            warn!("Failed to register insight {}: {}", insight.id, e);
                        }
                    } else {
                        warn!(
                            "Insight {} blocked by loop prevention: {:?}",
                            insight.id, validation_result.prevention_action
                        );
                    }
                }
                Err(e) => {
                    warn!("Failed to validate insight {}: {}", insight.id, e);
                }
            }
        }

        // Update metrics
        {
            let mut metrics = self.performance_metrics.write().await;
            metrics.total_reflections_completed += 1;
            metrics.total_insights_generated += session.generated_insights.len() as u64;
            metrics.last_updated = Utc::now();
        }

        info!(
            "Reflection completed: {} insights generated, {} clusters analyzed",
            session.generated_insights.len(),
            session.generated_clusters.len()
        );

        Ok(session)
    }

    /// Get current system performance metrics
    pub async fn get_performance_metrics(&self) -> CognitivePerformanceMetrics {
        let mut metrics = self.performance_metrics.read().await.clone();

        // Update uptime
        metrics.system_uptime_hours = Utc::now()
            .signed_duration_since(self.system_start_time)
            .num_seconds() as f64
            / 3600.0;

        metrics
    }

    /// Get loop prevention statistics
    pub async fn get_loop_prevention_statistics(
        &self,
    ) -> super::insight_loop_prevention::PreventionStatistics {
        self.loop_prevention_engine
            .read()
            .await
            .get_prevention_statistics()
    }

    /// Get background reflection service metrics
    pub async fn get_reflection_service_metrics(&self) -> ReflectionServiceMetrics {
        self.background_reflection_service.get_metrics().await
    }

    /// Manually trigger background reflection
    pub async fn trigger_background_reflection(&self, reason: String) -> Result<uuid::Uuid> {
        info!("Manually triggering background reflection: {}", reason);
        self.background_reflection_service
            .trigger_manual_reflection(reason)
            .await
    }

    /// Start the background reflection service
    pub async fn start_background_reflection(&self) -> Result<()> {
        if !self.background_reflection_service.is_running() {
            self.background_reflection_service.start().await?;
            info!("Background reflection service started");
        }
        Ok(())
    }

    /// Stop the background reflection service
    pub async fn stop_background_reflection(&self) -> Result<()> {
        if self.background_reflection_service.is_running() {
            self.background_reflection_service.stop().await?;
            info!("Background reflection service stopped");
        }
        Ok(())
    }

    /// Update system configuration
    pub async fn update_configuration(&mut self, config: CognitiveMemoryConfig) -> Result<()> {
        info!("Updating cognitive memory system configuration");

        // Validate configuration components
        config.scoring_config.validate()?;

        // Update engines
        self.scoring_engine
            .update_config(config.scoring_config.clone())?;

        // Update system config
        self.config = config;

        info!("Configuration updated successfully");
        Ok(())
    }

    /// Perform background maintenance tasks
    pub async fn background_maintenance(&self) -> Result<()> {
        if !self.config.enable_background_reflection {
            return Ok(());
        }

        let last_processing = *self.last_background_processing.read().await;
        let minutes_since_last = Utc::now()
            .signed_duration_since(last_processing)
            .num_minutes() as u64;

        if minutes_since_last < self.config.auto_processing_interval_minutes {
            return Ok(());
        }

        debug!("Performing background maintenance");

        // Check if reflection should be triggered
        if let Ok(should_reflect) = self
            .reflection_engine
            .read()
            .await
            .should_trigger_reflection()
            .await
        {
            if let Some(reason) = should_reflect {
                match self.trigger_reflection(reason).await {
                    Ok(_) => {
                        info!("Background reflection completed successfully");
                    }
                    Err(e) => {
                        warn!("Background reflection failed: {}", e);
                    }
                }
            }
        }

        // Update last processing time
        *self.last_background_processing.write().await = Utc::now();

        Ok(())
    }

    // Private helper methods

    async fn find_similar_memories(&self, memory: &Memory, limit: usize) -> Result<Vec<Memory>> {
        if memory.embedding.is_none() {
            return Ok(Vec::new());
        }

        let search_request = SearchRequest {
            query_embedding: Some(memory.embedding.as_ref().unwrap().as_slice().to_vec()),
            search_type: Some(SearchType::Semantic),
            similarity_threshold: Some(0.7),
            limit: Some(limit as i32),
            ..Default::default()
        };

        let search_response = self.repository.search_memories(search_request).await?;

        Ok(search_response
            .results
            .into_iter()
            .filter(|result| result.memory.id != memory.id)
            .map(|result| result.memory)
            .collect())
    }

    async fn update_performance_metrics(&self, processing_time_ms: u64, flags: &CognitiveFlags) {
        let mut metrics = self.performance_metrics.write().await;

        metrics.total_memories_processed += 1;

        // Update timing averages (simple moving average)
        let count = metrics.total_memories_processed as f64;
        metrics.average_scoring_time_ms =
            (metrics.average_scoring_time_ms * (count - 1.0) + processing_time_ms as f64) / count;

        if flags.consolidation_applied {
            metrics.average_consolidation_time_ms =
                (metrics.average_consolidation_time_ms * (count - 1.0) + processing_time_ms as f64)
                    / count;
        }

        metrics.last_updated = Utc::now();
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    async fn create_test_repository() -> Arc<MemoryRepository> {
        // This would create a test repository in real implementation
        panic!("Test repository creation not implemented - requires database setup");
    }

    #[tokio::test]
    #[ignore] // Ignore for now as it requires database setup
    async fn test_cognitive_memory_system_creation() {
        let repository = create_test_repository().await;
        let config = CognitiveMemoryConfig::default();

        let system = CognitiveMemorySystem::new(repository, config).await;
        assert!(system.is_ok());
    }

    #[test]
    fn test_cognitive_memory_config_defaults() {
        let config = CognitiveMemoryConfig::default();

        assert!(config.enable_auto_processing);
        assert!(config.enable_background_reflection);
        assert_eq!(config.auto_processing_interval_minutes, 30);
        assert_eq!(config.max_concurrent_operations, 10);
    }

    #[test]
    fn test_cognitive_flags_default() {
        let flags = CognitiveFlags::default();

        assert!(!flags.consolidation_applied);
        assert!(!flags.reflection_triggered);
        assert!(!flags.quality_validated);
        assert!(!flags.loop_prevention_checked);
        assert!(!flags.three_component_scored);
    }

    #[test]
    fn test_performance_metrics_default() {
        let metrics = CognitivePerformanceMetrics::default();

        assert_eq!(metrics.total_memories_processed, 0);
        assert_eq!(metrics.total_insights_generated, 0);
        assert_eq!(metrics.total_reflections_completed, 0);
        assert_eq!(metrics.average_scoring_time_ms, 0.0);
    }
}